Index

A

ACCESS_ERROR_MESSAGE (in module stages.text.classifiers.aegis_utils) actor_class (stages.deduplication.fuzzy.lsh.stage.LSHStage attribute) (stages.deduplication.shuffle_utils.stage.ShuffleStage attribute) actor_idle_time (utils.performance_utils.StagePerfStats attribute) add_filename_column (stages.text.download.base.extract.DocumentExtractStage attribute) (stages.text.download.base.iterator.DocumentIterateStage attribute) (stages.text.download.base.stage.DocumentDownloadExtractStage attribute) add_stage() (pipeline.pipeline.Pipeline method) add_stage_perf() (tasks.tasks.Task method) AddId (class in stages.text.modules.add_id) AEGIS_LABELS (in module stages.text.classifiers.aegis_utils) aegis_variant (stages.text.classifiers.aegis.AegisClassifier attribute) AEGIS_VARIANTS (in module stages.text.classifiers.aegis) AegisClassifier (class in stages.text.classifiers.aegis) AegisModel (class in stages.text.classifiers.aegis) AegisModelStage (class in stages.text.classifiers.aegis) aesthetic_score (tasks.image.ImageObject attribute) (tasks.video.Clip attribute) AESTHETICS (utils.decoder_utils.FramePurpose attribute) align_down_to_256() (in module stages.deduplication.gpu_utils) allocation (backends.base.WorkerMetadata attribute)

AlphaFilter (class in stages.text.filters.code) append_mode_implemented (stages.text.io.writer.base.BaseWriter attribute) ArxivDownloader (class in stages.text.download.arxiv.download) ArxivDownloadExtractStage (class in stages.text.download.arxiv.stage) ArxivExtractor (class in stages.text.download.arxiv.extract) ArxivIterator (class in stages.text.download.arxiv.iterator) ArxivUrlGenerator (class in stages.text.download.arxiv.url_generation) as_posix() (utils.client_utils.FSPath method) asr_model (stages.audio.inference.asr_nemo.InferenceAsrNemoStage attribute) assign_id() (stages.deduplication.io_utils.DeduplicationIO method) ATTENTION_MASK_COLUMN (in module stages.text.models.utils) audio_codec (tasks.video.VideoMetadata attribute) (utils.decoder_utils.VideoMetadata attribute) audio_filepath_key (stages.audio.common.GetAudioDurationStage attribute) AudioBatch (class in tasks.audio_batch) AudioToDocumentStage (class in stages.audio.io.convert) autocast (stages.text.classifiers.aegis.AegisClassifier attribute) (stages.text.classifiers.aegis.InstructionDataGuardClassifier attribute) (stages.text.classifiers.base.DistributedDataClassifier attribute) (stages.text.classifiers.prompt_task_complexity.PromptTaskComplexityClassifier attribute) (stages.text.embedders.base.EmbeddingCreatorStage attribute)

B

backends module backends.base module backends.experimental module backends.experimental.ray_actor_pool module backends.experimental.ray_actor_pool.adapter module backends.experimental.ray_actor_pool.executor module backends.experimental.ray_actor_pool.raft_adapter module backends.experimental.ray_actor_pool.shuffle_adapter module backends.experimental.ray_actor_pool.utils module backends.experimental.ray_data module backends.experimental.ray_data.adapter module backends.experimental.ray_data.executor module backends.experimental.ray_data.utils module backends.experimental.utils module backends.internal module backends.internal.raft module backends.internal.raft.ray_comms module

backends.utils module backends.xenna module backends.xenna.adapter module backends.xenna.executor module bands_per_iteration (stages.deduplication.fuzzy.lsh.stage.LSHStage attribute) BaseCommonCrawlUrlGenerator (class in stages.text.download.common_crawl.url_generation) BaseExecutor (class in backends.base) BaseFilterStage (class in stages.image.filters.base) BaseReader (class in stages.text.io.reader.base) BaseStageAdapter (class in backends.base) BaseWriter (class in stages.text.io.writer.base) batch_size (backends.experimental.ray_data.adapter.RayDataStageAdapter property) (stages.base.ProcessingStage property) bit_rate_k (tasks.video.VideoMetadata attribute) (utils.decoder_utils.VideoMetadata attribute) blocksize (stages.file_partitioning.FilePartitioningStage attribute) (stages.text.io.reader.jsonl.JsonlReader attribute) (stages.text.io.reader.parquet.ParquetReader attribute) BoilerPlateStringFilter (class in stages.text.filters.heuristic_filter) BoilerPlateStringModifier (class in stages.text.modifiers.c4) break_parquet_partition_into_groups() (in module stages.deduplication.semantic.utils) broadcast_root_unique_id() (backends.experimental.ray_actor_pool.raft_adapter.RayActorPoolRAFTAdapter method) BucketsToEdgesStage (class in stages.deduplication.fuzzy.buckets_to_edges) buffer (tasks.video.Clip attribute) build() (pipeline.pipeline.Pipeline method) BulkRapidsMPFShuffler (class in stages.deduplication.shuffle_utils.rapidsmpf_shuffler) bullet_list (in module stages.text.utils.constants) BulletsFilter (class in stages.text.filters.heuristic_filter)

C

cache_dir (stages.text.classifiers.aegis.AegisClassifier attribute) (stages.text.classifiers.aegis.InstructionDataGuardClassifier attribute) (stages.text.classifiers.aegis.PostProcessAegisResponsesStage attribute) (stages.text.classifiers.base.DistributedDataClassifier attribute) (stages.text.classifiers.prompt_task_complexity.PromptTaskComplexityClassifier attribute) cache_kwargs (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) cache_path (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) calculate_concurrency_for_actors_for_stage() (in module backends.experimental.ray_data.utils) calculate_optimal_actors_for_stage() (in module backends.experimental.ray_actor_pool.utils) calculate_sha256() (stages.video.io.clip_writer.ClipWriterStage static method) caption_batch_size (stages.video.caption.caption_generation.CaptionGenerationStage attribute) caption_models (stages.video.io.clip_writer.ClipWriterStage attribute) CaptionEnhancementStage (class in stages.video.caption.caption_enhancement) CaptionGenerationStage (class in stages.video.caption.caption_generation) CaptionPreparationStage (class in stages.video.caption.caption_preparation) CAT_ALIASES (in module stages.text.download.wikipedia.extract) ceil_by_factor() (in module utils.windowing_utils) check_cuda() (stages.audio.inference.asr_nemo.InferenceAsrNemoStage method) check_disallowed_kwargs() (in module utils.file_utils) check_if_small_motion() (in module stages.video.filtering.motion_vector_backend) check_output_mode() (in module utils.file_utils) ClassifierModelStage (class in stages.text.classifiers.base) cleanup() (stages.deduplication.shuffle_utils.rapidsmpf_shuffler.BulkRapidsMPFShuffler method) clear_output (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) ClientPartitioningStage (class in stages.client_partitioning) Clip (class in tasks.video) clip_chunk_index (tasks.video.Video attribute) clip_len_s (stages.video.clipping.clip_extraction_stages.FixedStrideExtractorStage attribute) clip_stats (tasks.video.Video attribute) clip_stride_s (stages.video.clipping.clip_extraction_stages.FixedStrideExtractorStage attribute) clip_tokens() (in module stages.text.models.utils) ClipAestheticFilterStage (class in stages.video.filtering.clip_aesthetic_filter) ClipFrameExtractionStage (class in stages.video.clipping.clip_frame_extraction) clips (tasks.video.Video attribute) ClipStats (class in tasks.video) ClipTranscodingStage (class in stages.video.clipping.clip_extraction_stages) ClipWriterStage (class in stages.video.io.clip_writer) ClusterWiseFilePartitioningStage (class in stages.deduplication.semantic.pairwise_io) collect_outputs() (stages.text.embedders.base.EmbeddingModelStage method) (stages.text.models.model.ModelStage method) collect_stage_metrics() (tasks.utils.TaskPerfUtils static method) combine() (tasks.video.ClipStats method) common_english_words (in module stages.text.utils.constants) CommonCrawlDownloadExtractStage (class in stages.text.download.common_crawl.stage) CommonCrawlHTMLExtractor (class in stages.text.download.common_crawl.extract) CommonCrawlWARCDownloader (class in stages.text.download.common_crawl.download) CommonCrawlWarcIterator (class in stages.text.download.common_crawl.warc_iterator)

CommonEnglishWordsFilter (class in stages.text.filters.heuristic_filter) Comms (class in backends.internal.raft.ray_comms) CompositeStage (class in stages.base) compression_level (stages.video.preview.preview.PreviewStage attribute) compute_filter_mask() (stages.text.modules.score_filter.Filter method) (stages.text.modules.score_filter.ScoreFilter method) compute_minhashes() (stages.deduplication.fuzzy.minhash.GPUMinHash method) (stages.deduplication.fuzzy.minhash.MinHash method) compute_results() (stages.text.classifiers.prompt_task_complexity.CustomDeberta method) compute_windows() (in module utils.windowing_utils) configure_forward() (stages.text.classifiers.fineweb_edu.FineWebModelStage static method) ConnectedComponentsStage (class in stages.deduplication.fuzzy.connected_components) construct_base_name() (stages.image.io.image_writer.ImageWriterStage method) CONTENT_TYPE_MODEL_IDENTIFIER (in module stages.text.classifiers.content_type) ContentTypeClassifier (class in stages.text.classifiers.content_type) ConvertImageBatchToDocumentBatchStage (class in stages.image.io.convert) COSINE_DIST_TO_CENT_COL (in module stages.deduplication.semantic.kmeans) cosmos_embed1_embedding (tasks.video.Clip attribute) cosmos_embed1_frames (tasks.video.Clip attribute) cosmos_embed1_text_match (tasks.video.Clip attribute) CosmosEmbed1EmbeddingStage (class in stages.video.embedding.cosmos_embed1) CosmosEmbed1FrameCreationStage (class in stages.video.embedding.cosmos_embed1) cpus (stages.resources.Resources attribute) create_actor_from_stage() (in module backends.experimental.ray_data.adapter) create_id_generator_actor() (in module stages.deduplication.id_generator) create_list_series_from_1d_or_2d_ar() (in module stages.text.embedders.utils) create_named_ray_actor_pool_stage_adapter() (in module backends.experimental.ray_actor_pool.utils) create_named_xenna_stage_adapter() (in module backends.xenna.adapter) create_nccl_uniqueid() (backends.internal.raft.ray_comms.Comms method) create_or_overwrite_dir() (in module utils.file_utils) create_output_dataframe() (stages.text.classifiers.aegis.AegisModelStage method) (stages.text.classifiers.base.ClassifierModelStage method) (stages.text.classifiers.fineweb_edu.FineWebModelStage method) (stages.text.classifiers.prompt_task_complexity.PromptTaskComplexityModelStage method) (stages.text.embedders.base.EmbeddingModelStage method) (stages.text.models.model.ModelStage method) create_task_from_stage() (in module backends.experimental.ray_data.adapter) CreateInitialManifestFleursStage (class in stages.audio.datasets.fleurs.create_initial_manifest) crop_s (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage attribute) CURATOR_DEDUP_ID_STR (in module stages.deduplication.id_generator) CURATOR_DEFAULT_MINHASH_FIELD (in module stages.deduplication.fuzzy.utils) CURATOR_FUZZY_DUPLICATE_GROUP_FIELD (in module stages.deduplication.fuzzy.utils) CURATOR_ID_GENERATOR_ACTOR_NAME (in module stages.deduplication.id_generator) CURATOR_LSH_BUCKET_FIELD (in module stages.deduplication.fuzzy.utils) custom_metrics (utils.performance_utils.StagePerfStats attribute) custom_read() (stages.deduplication.io_utils.DeduplicationIO method) CustomDeberta (class in stages.text.classifiers.prompt_task_complexity)

D

data (tasks.document.DocumentBatch attribute) (tasks.file_group.FileGroupTask attribute) (tasks.image.ImageBatch attribute) (tasks.tasks.Task attribute) (tasks.video.VideoTask attribute) data_prefix (stages.text.download.common_crawl.url_generation.BaseCommonCrawlUrlGenerator attribute) dataset_name (tasks.tasks.Task attribute) Deberta (class in stages.text.classifiers.base) DEBERTA_TOKENIZER_PADDING_SIDE (in module stages.text.classifiers.constants) decode_for_motion() (in module stages.video.filtering.motion_vector_backend) decode_html() (in module stages.text.download.utils) decode_video_cpu() (in module utils.decoder_utils) decode_video_cpu_frame_ids() (in module utils.decoder_utils) decoded_motion_data (tasks.video.Clip attribute) DecodedData (class in stages.video.filtering.motion_vector_backend) decoder_mode (stages.video.clipping.video_frame_extraction.VideoFrameExtractionStage attribute) decompose() (stages.base.CompositeStage method) (stages.deduplication.semantic.kmeans.KMeansStage method) (stages.deduplication.semantic.pairwise.PairwiseStage method) (stages.text.classifiers.aegis.AegisClassifier method) (stages.text.classifiers.aegis.InstructionDataGuardClassifier method) (stages.text.classifiers.base.DistributedDataClassifier method) (stages.text.classifiers.prompt_task_complexity.PromptTaskComplexityClassifier method) (stages.text.download.arxiv.stage.ArxivDownloadExtractStage method) (stages.text.download.base.stage.DocumentDownloadExtractStage method) (stages.text.download.common_crawl.stage.CommonCrawlDownloadExtractStage method) (stages.text.download.wikipedia.stage.WikipediaDownloadExtractStage method) (stages.text.embedders.base.EmbeddingCreatorStage method) (stages.text.io.reader.jsonl.JsonlReader method) (stages.text.io.reader.parquet.ParquetReader method) (stages.video.io.video_reader.VideoReader method) decompose_and_apply_with() (stages.base.CompositeStage method) DeduplicationIO (class in stages.deduplication.io_utils) default() (utils.writer_utils.JsonEncoderCustom method) delete_dir() (in module utils.file_utils)

describe() (pipeline.pipeline.Pipeline method) detect_language() (in module stages.text.download.utils) deterministic_name (stages.image.io.image_writer.ImageWriterStage attribute) device (stages.text.classifiers.aegis.AegisModel property) (stages.text.classifiers.base.Deberta property) (stages.text.classifiers.prompt_task_complexity.CustomDeberta property) disable_mmcache (stages.video.caption.caption_generation.CaptionGenerationStage attribute) distance_metric (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) DistributedDataClassifier (class in stages.text.classifiers.base) DocumentBatch (class in tasks.document) DocumentDownloader (class in stages.text.download.base.download) DocumentDownloadExtractStage (class in stages.text.download.base.stage) DocumentDownloadStage (class in stages.text.download.base.download) DocumentExtractor (class in stages.text.download.base.extract) DocumentExtractStage (class in stages.text.download.base.extract) DocumentFilter (class in stages.text.filters.doc_filter) DocumentIterateStage (class in stages.text.download.base.iterator) DocumentIterator (class in stages.text.download.base.iterator) DocumentModifier (class in stages.text.modifiers.doc_modifier) DOMAIN_MODEL_IDENTIFIER (in module stages.text.classifiers.domain) DomainClassifier (class in stages.text.classifiers.domain) download() (stages.text.download.base.download.DocumentDownloader method) download_extract_files() (stages.audio.datasets.fleurs.create_initial_manifest.CreateInitialManifestFleursStage method) download_file() (in module stages.audio.datasets.file_utils) download_model_from_hf() (in module utils.hf_download_utils) downloader (stages.text.download.base.download.DocumentDownloadStage attribute) (stages.text.download.base.stage.DocumentDownloadExtractStage attribute) dry_run (stages.video.io.clip_writer.ClipWriterStage attribute) dump_date (stages.text.download.wikipedia.url_generation.WikipediaUrlGenerator attribute) duplicate_id_field (stages.image.deduplication.removal.ImageDuplicatesRemovalStage attribute) (stages.text.deduplication.removal.TextDuplicatesRemovalStage attribute) DUPLICATE_IDS_SUBDIR (in module stages.deduplication.fuzzy.identify_duplicates) duration (tasks.video.Clip property) (tasks.video.VideoMetadata attribute) duration_key (stages.audio.common.GetAudioDurationStage attribute)

E

egomotion (tasks.video.Clip attribute) ellipsis_marks (in module stages.text.utils.constants) EllipsisFilter (class in stages.text.filters.heuristic_filter) embedding (tasks.image.ImageObject attribute) embedding_algorithm (stages.video.io.clip_writer.ClipWriterStage attribute) embedding_dim (stages.deduplication.semantic.kmeans.KMeansStage attribute) (stages.deduplication.semantic.pairwise.PairwiseStage attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) embedding_field (stages.deduplication.semantic.kmeans.KMeansStage attribute) (stages.deduplication.semantic.pairwise.PairwiseStage attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) (stages.text.embedders.base.EmbeddingCreatorStage attribute) embedding_max_chars (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) embedding_max_seq_length (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) embedding_model_inference_batch_size (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) embedding_padding_side (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) embedding_pooling (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) (stages.text.embedders.base.EmbeddingCreatorStage attribute) EmbeddingCreatorStage (class in stages.text.embedders.base) EmbeddingModelStage (class in stages.text.embedders.base) EMBEDDINGS (utils.decoder_utils.FramePurpose attribute) EmptyTask (in module tasks.tasks) enable_statistics (stages.deduplication.fuzzy.lsh.stage.LSHStage attribute) encode_batch_size (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage attribute) encoder (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage attribute) encoder_threads (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage attribute) end (utils.windowing_utils.WindowFrameInfo attribute) end_marks (in module stages.text.utils.constants) end_snapshot_str (stages.text.download.common_crawl.url_generation.BaseCommonCrawlUrlGenerator attribute) enhanced_caption_models (stages.video.io.clip_writer.ClipWriterStage attribute) entire_gpu (stages.resources.Resources attribute) entire_scene_as_clip (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage attribute) env_info (backends.xenna.adapter.XennaStageAdapter property) eps (stages.deduplication.semantic.identify_duplicates.IdentifyDuplicatesStage attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) errors (tasks.video.Clip attribute) (tasks.video.Video attribute)

EXACT_DUPLICATE_GROUP_FIELD (in module stages.deduplication.exact.identification) ExactDeduplicationWorkflow (class in stages.deduplication.exact.workflow) ExactDuplicateIdentification (class in stages.deduplication.exact.identification) execute() (backends.base.BaseExecutor method) (backends.experimental.ray_actor_pool.executor.RayActorPoolExecutor method) (backends.experimental.ray_data.executor.RayDataExecutor method) (backends.xenna.executor.XennaExecutor method) execute_setup_on_node() (in module backends.experimental.utils) extract() (stages.deduplication.shuffle_utils.rapidsmpf_shuffler.BulkRapidsMPFShuffler method) (stages.text.download.arxiv.extract.ArxivExtractor method) (stages.text.download.base.extract.DocumentExtractor method) (stages.text.download.common_crawl.extract.CommonCrawlHTMLExtractor method) (stages.text.download.wikipedia.extract.WikipediaExtractor method) extract_and_group() (stages.deduplication.fuzzy.lsh.lsh.LSHActor method) extract_and_write() (backends.experimental.ray_actor_pool.shuffle_adapter.ShuffleStageAdapter method) (stages.deduplication.exact.identification.ExactDuplicateIdentification method) (stages.deduplication.fuzzy.identify_duplicates.IdentifyDuplicatesStage method) (stages.deduplication.fuzzy.lsh.lsh.LSHActor method) (stages.deduplication.fuzzy.lsh.stage.LSHStage method) (stages.deduplication.shuffle_utils.rapidsmpf_shuffler.BulkRapidsMPFShuffler method) (stages.deduplication.shuffle_utils.stage.ShuffleStage method) extract_archive() (in module stages.audio.datasets.file_utils) extract_frames() (in module utils.decoder_utils) extract_metadata() (tasks.video.Clip method) extract_purposes (stages.video.clipping.clip_frame_extraction.ClipFrameExtractionStage attribute) extract_text() (stages.text.download.html_extractors.base.HTMLExtractorAlgorithm method) (stages.text.download.html_extractors.justext.JusTextExtractor method) (stages.text.download.html_extractors.resiliparse.ResiliparseExtractor method) (stages.text.download.html_extractors.trafilatura.TrafilaturaExtractor method) extract_video_metadata() (in module utils.decoder_utils) extracted_frames (tasks.video.Clip attribute) extraction_policies (stages.video.clipping.clip_frame_extraction.ClipFrameExtractionStage attribute) extraction_policy (utils.decoder_utils.FrameExtractionSignature attribute) extractor (stages.text.download.base.extract.DocumentExtractStage attribute) (stages.text.download.base.stage.DocumentDownloadExtractStage attribute)

F

FastTextLabelModifier (class in stages.text.modifiers.fasttext) FastTextLangId (class in stages.text.filters.fasttext_filter) FastTextQualityFilter (class in stages.text.filters.fasttext_filter) fetch_video() (in module utils.windowing_utils) ffmpeg_verbose (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage attribute) fields (stages.image.io.convert.ConvertImageBatchToDocumentBatchStage attribute) (stages.text.io.reader.base.BaseReader attribute) (stages.text.io.reader.jsonl.JsonlReader attribute) (stages.text.io.reader.parquet.ParquetReader attribute) (stages.text.io.writer.base.BaseWriter attribute) file_extension (stages.text.io.writer.base.BaseWriter attribute) (stages.text.io.writer.jsonl.JsonlWriter attribute) (stages.text.io.writer.parquet.ParquetWriter attribute) file_extensions (stages.file_partitioning.FilePartitioningStage attribute) (stages.text.io.reader.jsonl.JsonlReader attribute) (stages.text.io.reader.parquet.ParquetReader attribute) file_paths (stages.file_partitioning.FilePartitioningStage attribute) (stages.text.io.reader.jsonl.JsonlReader attribute) (stages.text.io.reader.parquet.ParquetReader attribute) FileGroupTask (class in tasks.file_group) FilePartitioningStage (class in stages.file_partitioning) filepath_key (stages.audio.datasets.fleurs.create_initial_manifest.CreateInitialManifestFleursStage attribute) (stages.audio.inference.asr_nemo.InferenceAsrNemoStage attribute) files_per_partition (stages.file_partitioning.FilePartitioningStage attribute) (stages.text.io.reader.jsonl.JsonlReader attribute) (stages.text.io.reader.parquet.ParquetReader attribute) FILETYPE_TO_DEFAULT_EXTENSIONS (in module utils.file_utils) Filter (class in stages.text.modules.score_filter) filter_by (stages.text.classifiers.aegis.AegisClassifier attribute) (stages.text.classifiers.aegis.InstructionDataGuardClassifier attribute) (stages.text.classifiers.base.DistributedDataClassifier attribute) (stages.text.classifiers.prompt_task_complexity.PromptTaskComplexityClassifier attribute) filter_by_category() (stages.text.classifiers.aegis.AegisClassifier method) (stages.text.classifiers.aegis.InstructionDataGuardClassifier method) (stages.text.classifiers.base.DistributedDataClassifier method) filter_field (stages.text.modules.score_filter.Filter attribute) filter_files_by_extension() (in module utils.file_utils) filter_fn (stages.text.modules.score_filter.Filter attribute) filter_obj (stages.text.modules.score_filter.ScoreFilter attribute) filtered_clips (tasks.video.Video attribute)

find_closest_indices() (in module utils.decoder_utils) FINEWEB_EDU_MODEL_IDENTIFIER (in module stages.text.classifiers.fineweb_edu) FINEWEB_MIXTRAL_EDU_MODEL_IDENTIFIER (in module stages.text.classifiers.fineweb_edu) FINEWEB_NEMOTRON_EDU_MODEL_IDENTIFIER (in module stages.text.classifiers.fineweb_edu) FineWebEduClassifier (class in stages.text.classifiers.fineweb_edu) FineWebMixtralEduClassifier (class in stages.text.classifiers.fineweb_edu) FineWebModelStage (class in stages.text.classifiers.fineweb_edu) FineWebNemotronEduClassifier (class in stages.text.classifiers.fineweb_edu) first (utils.decoder_utils.FrameExtractionPolicy attribute) FixedStrideExtractorStage (class in stages.video.clipping.clip_extraction_stages) floor_by_factor() (in module utils.windowing_utils) format_aegis() (in module stages.text.classifiers.aegis_utils) format_name_with_suffix() (in module stages.text.models.utils) FormatAegisPromptStage (class in stages.text.classifiers.aegis) forward() (stages.text.classifiers.aegis.AegisModel method) (stages.text.classifiers.aegis.InstructionDataGuardNet method) (stages.text.classifiers.base.Deberta method) (stages.text.classifiers.prompt_task_complexity.CustomDeberta method) (stages.text.classifiers.prompt_task_complexity.MeanPooling method) (stages.text.classifiers.prompt_task_complexity.MulticlassHead method) fp8 (stages.video.caption.caption_enhancement.CaptionEnhancementStage attribute) (stages.video.caption.caption_generation.CaptionGenerationStage attribute) FPS (in module utils.windowing_utils) fps (utils.decoder_utils.VideoMetadata attribute) (utils.nvcodec_utils.FrameExtractionPolicy attribute) FPS_MAX_FRAMES (in module utils.windowing_utils) FPS_MIN_FRAMES (in module utils.windowing_utils) fraction (tasks.video.Video property) frame_array (tasks.video.Video attribute) FRAME_FACTOR (in module utils.windowing_utils) frame_size (stages.video.filtering.motion_vector_backend.DecodedData attribute) FrameExtractionPolicy (class in utils.decoder_utils) (class in utils.nvcodec_utils) FrameExtractionSignature (class in utils.decoder_utils) FramePurpose (class in utils.decoder_utils) framerate (tasks.video.VideoMetadata attribute) frames (stages.video.filtering.motion_vector_backend.DecodedData attribute) from_disk() (stages.deduplication.id_generator.IdGeneratorBase class method) FSPath (class in utils.client_utils) full (utils.nvcodec_utils.FrameExtractionPolicy attribute) FuzzyDeduplicationWorkflow (class in stages.deduplication.fuzzy.workflow)

G

GeneralCommentToCodeFilter (class in stages.text.filters.code) generate_captions (stages.video.io.clip_writer.ClipWriterStage attribute) generate_data_urls() (stages.text.download.common_crawl.url_generation.BaseCommonCrawlUrlGenerator method) generate_decoded_frames() (utils.nvcodec_utils.NvVideoDecoder method) generate_embeddings (stages.video.io.clip_writer.ClipWriterStage attribute) generate_path_urls() (stages.text.download.common_crawl.url_generation.BaseCommonCrawlUrlGenerator method) (stages.text.download.common_crawl.url_generation.MainCommonCrawlUrlGenerator method) (stages.text.download.common_crawl.url_generation.NewsCommonCrawlUrlGenerator method) generate_previews (stages.video.caption.caption_preparation.CaptionPreparationStage attribute) (stages.video.io.clip_writer.ClipWriterStage attribute) generate_seeds() (stages.deduplication.fuzzy.minhash.GPUMinHash method) (stages.deduplication.fuzzy.minhash.MinHash method) generate_stage2_caption (stages.video.caption.caption_generation.CaptionGenerationStage attribute) generate_urls() (stages.text.download.arxiv.url_generation.ArxivUrlGenerator method) (stages.text.download.base.url_generation.URLGenerator method) (stages.text.download.common_crawl.url_generation.BaseCommonCrawlUrlGenerator method) (stages.text.download.wikipedia.url_generation.WikipediaUrlGenerator method) get_all_file_paths_and_size_under() (in module utils.file_utils) get_all_file_paths_under() (in module utils.file_utils) get_array_from_df() (in module stages.deduplication.semantic.utils) get_available_cpu_gpu_resources() (in module backends.experimental.utils) get_avg_frame_rate() (in module utils.decoder_utils) get_band_iterations() (stages.deduplication.fuzzy.lsh.stage.LSHStage method) get_batch_range() (stages.deduplication.id_generator.IdGeneratorBase method) get_batch_size() (backends.experimental.ray_actor_pool.adapter.RayActorPoolStageAdapter method) (backends.experimental.ray_actor_pool.raft_adapter.RayActorPoolRAFTAdapter method) (backends.experimental.ray_actor_pool.shuffle_adapter.ShuffleStageAdapter method) get_bytes_cat_ranges() (utils.client_utils.FSPath method) get_cer() (in module stages.audio.metrics.get_wer) get_charrate() (in module stages.audio.metrics.get_wer) get_columns() (tasks.document.DocumentBatch method) get_comments() (in module stages.text.utils.text_utils) get_comments_and_docstring() (in module stages.text.utils.text_utils) get_config() (stages.base.ProcessingStage method) get_description() (stages.base.CompositeStage method) (stages.text.download.arxiv.stage.ArxivDownloadExtractStage method) (stages.text.download.base.stage.DocumentDownloadExtractStage method) (stages.text.download.common_crawl.stage.CommonCrawlDownloadExtractStage method) (stages.text.download.wikipedia.stage.WikipediaDownloadExtractStage method) (stages.text.io.reader.jsonl.JsonlReader method) (stages.text.io.reader.parquet.ParquetReader method) (stages.video.io.video_reader.VideoReader method) get_deterministic_hash() (in module stages.text.io.writer.utils) get_device_free_memory() (in module stages.deduplication.gpu_utils) get_docstrings() (in module stages.text.utils.text_utils)

get_file_extension() (stages.text.io.writer.base.BaseWriter method) get_fleurs_url_list() (in module stages.audio.datasets.fleurs.create_initial_manifest) get_fps() (utils.nvcodec_utils.VideoBatchDecoder method) get_frame_count() (in module utils.decoder_utils) get_frames_from_ffmpeg() (in module stages.video.clipping.video_frame_extraction) get_fs() (in module utils.file_utils) get_full_path() (in module utils.storage_utils) get_id_generator_actor() (in module stages.deduplication.id_generator) get_major_size() (stages.video.filtering.motion_vector_backend.DecodedData method) (tasks.video.Clip method) (tasks.video.Video method) get_next_frames() (utils.nvcodec_utils.NvVideoDecoder method) get_ngrams() (in module stages.text.utils.text_utils) get_output_path_ce1_embd() (stages.video.io.clip_writer.ClipWriterStage static method) get_output_path_ce1_embd_parquet() (stages.video.io.clip_writer.ClipWriterStage static method) get_output_path_clips() (stages.video.io.clip_writer.ClipWriterStage static method) get_output_path_iv2_embd() (stages.video.io.clip_writer.ClipWriterStage static method) get_output_path_iv2_embd_parquet() (stages.video.io.clip_writer.ClipWriterStage static method) get_output_path_metas() (stages.video.io.clip_writer.ClipWriterStage static method) get_output_path_previews() (stages.video.io.clip_writer.ClipWriterStage static method) get_output_path_processed_clip_chunks() (stages.video.io.clip_writer.ClipWriterStage static method) get_output_path_processed_videos() (stages.video.io.clip_writer.ClipWriterStage static method) get_paragraphs() (in module stages.text.utils.text_utils) get_sentences() (in module stages.text.utils.text_utils) get_stage_class() (in module stages.base) get_stop_list_dict() (in module stages.text.download.html_extractors.utils) get_tmp_dir() (in module utils.operation_utils) get_video_timestamps() (in module utils.decoder_utils) get_wer() (in module stages.audio.metrics.get_wer) get_word_splitter() (in module stages.text.utils.text_utils) get_wordrate() (in module stages.audio.metrics.get_wer) get_words() (in module stages.text.utils.text_utils) get_worker_metadata_and_node_id() (in module backends.experimental.utils) GetAudioDurationStage (class in stages.audio.common) GetPairwiseWerStage (class in stages.audio.metrics.get_wer) global_mean (stages.video.filtering.motion_vector_backend.MotionInfo attribute) global_mean_threshold (stages.video.filtering.motion_filter.MotionFilterStage attribute) gpu_decode_for_stitching() (in module utils.nvcodec_utils) gpu_memory_gb (stages.resources.Resources attribute) (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage attribute) (stages.video.embedding.cosmos_embed1.CosmosEmbed1EmbeddingStage attribute) (stages.video.embedding.internvideo2.InternVideo2EmbeddingStage attribute) GPUMinHash (class in stages.deduplication.fuzzy.minhash) gpus (stages.resources.Resources attribute) group_by_bucket() (stages.deduplication.fuzzy.lsh.lsh.LSHActor method)

H

has_metadata() (tasks.video.Video method) hash_files() (stages.deduplication.id_generator.IdGeneratorBase method) height (tasks.video.VideoMetadata attribute) (utils.decoder_utils.Resolution attribute) (utils.decoder_utils.VideoMetadata attribute) hf_token (stages.text.classifiers.aegis.AegisClassifier attribute) (stages.text.classifiers.aegis.InstructionDataGuardClassifier attribute) (stages.text.classifiers.aegis.PostProcessAegisResponsesStage attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) (stages.text.embedders.base.EmbeddingCreatorStage attribute)

HIDDEN_TEXT_COLUMN (in module stages.text.classifiers.aegis) HistogramFilter (class in stages.text.filters.heuristic_filter) HTMLBoilerplateFilter (class in stages.text.filters.code) HTMLExtractorAlgorithm (class in stages.text.download.html_extractors.base)

I

id_field (stages.deduplication.fuzzy.lsh.stage.LSHStage attribute) (stages.deduplication.semantic.kmeans.KMeansStage attribute) (stages.deduplication.semantic.pairwise.PairwiseStage attribute) (stages.text.deduplication.removal.TextDuplicatesRemovalStage attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) (stages.text.modules.add_id.AddId attribute) ID_GENERATOR_OUTPUT_FILENAME (in module stages.deduplication.exact.workflow) (in module stages.deduplication.fuzzy.workflow) id_generator_path (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) id_generator_state_file (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) id_generator_storage_options (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) id_prefix (stages.text.modules.add_id.AddId attribute) IdentifyDuplicatesStage (class in stages.deduplication.fuzzy.identify_duplicates) (class in stages.deduplication.semantic.identify_duplicates) IdGenerator (class in stages.deduplication.id_generator) IdGeneratorBase (class in stages.deduplication.id_generator) ids_to_remove_duplicate_id_field (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) ids_to_remove_path (stages.text.deduplication.removal.TextDuplicatesRemovalStage attribute) (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) ids_to_remove_read_kwargs (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) image_data (tasks.image.ImageObject attribute) IMAGE_FACTOR (in module utils.windowing_utils) image_id (tasks.image.ImageObject attribute) image_path (tasks.image.ImageObject attribute) ImageAestheticFilterStage (class in stages.image.filters.aesthetic_filter) ImageBatch (class in tasks.image) ImageDuplicatesRemovalStage (class in stages.image.deduplication.removal) ImageEmbeddingStage (class in stages.image.embedders.clip_embedder) ImageNSFWFilterStage (class in stages.image.filters.nsfw_filter) ImageObject (class in tasks.image) ImageReaderStage (class in stages.image.io.image_reader) images_per_tar (stages.image.io.image_writer.ImageWriterStage attribute) ImageWriterStage (class in stages.image.io.image_writer) index_prefix (stages.text.download.common_crawl.url_generation.MainCommonCrawlUrlGenerator attribute) infer_dataset_name_from_path() (in module utils.file_utils) infer_protocol_from_paths() (in module utils.file_utils) InferenceAsrNemoStage (class in stages.audio.inference.asr_nemo) init (stages.deduplication.semantic.kmeans.KMeansStage attribute) init() (backends.internal.raft.ray_comms.Comms method) input_blocksize (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) input_columns() (stages.text.download.arxiv.extract.ArxivExtractor method) (stages.text.download.base.extract.DocumentExtractor method) (stages.text.download.common_crawl.extract.CommonCrawlHTMLExtractor method) (stages.text.download.wikipedia.extract.WikipediaExtractor method) input_data_size_mb (utils.performance_utils.StagePerfStats attribute) input_fields (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) (stages.text.modules.modifier.Modify attribute) input_file_extensions (stages.deduplication.semantic.kmeans.KMeansStage attribute) (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) input_files_per_partition (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) input_filetype (stages.deduplication.semantic.kmeans.KMeansStage attribute) (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) INPUT_ID_COLUMN (in module stages.text.models.utils) input_id_field (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) input_kwargs (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) input_list_json_path (stages.client_partitioning.ClientPartitioningStage attribute) input_path (stages.deduplication.semantic.kmeans.KMeansStage attribute) (stages.deduplication.semantic.pairwise.PairwiseStage attribute) (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) (stages.video.io.clip_writer.ClipWriterStage attribute) (stages.video.io.video_reader.VideoReaderStage attribute) (tasks.video.Video property) input_task_limit (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) input_video (tasks.video.Video attribute) input_video_path (stages.video.io.video_reader.VideoReader attribute) inputs() (stages.audio.inference.asr_nemo.InferenceAsrNemoStage method) (stages.base.CompositeStage method) (stages.base.ProcessingStage method) (stages.deduplication.exact.identification.ExactDuplicateIdentification method) (stages.deduplication.fuzzy.minhash.MinHashStage method) (stages.deduplication.semantic.pairwise_io.ClusterWiseFilePartitioningStage method) (stages.file_partitioning.FilePartitioningStage method) (stages.image.deduplication.removal.ImageDuplicatesRemovalStage method) (stages.image.embedders.clip_embedder.ImageEmbeddingStage method) (stages.image.filters.base.BaseFilterStage method) (stages.image.io.image_reader.ImageReaderStage method) (stages.image.io.image_writer.ImageWriterStage method) (stages.text.classifiers.aegis.AegisClassifier method) (stages.text.classifiers.aegis.FormatAegisPromptStage method) (stages.text.classifiers.aegis.InstructionDataGuardClassifier method) (stages.text.classifiers.aegis.PostProcessAegisResponsesStage method) (stages.text.classifiers.base.DistributedDataClassifier method) (stages.text.classifiers.prompt_task_complexity.PromptTaskComplexityClassifier method) (stages.text.deduplication.removal.TextDuplicatesRemovalStage method) (stages.text.download.base.download.DocumentDownloadStage method) (stages.text.download.base.extract.DocumentExtractStage method) (stages.text.download.base.iterator.DocumentIterateStage method) (stages.text.download.base.url_generation.URLGenerationStage method) (stages.text.io.reader.base.BaseReader method) (stages.text.io.writer.base.BaseWriter method) (stages.text.models.model.ModelStage method) (stages.text.models.tokenizer.TokenizerStage method) (stages.text.modules.add_id.AddId method) (stages.text.modules.modifier.Modify method) (stages.text.modules.score_filter.Filter method) (stages.text.modules.score_filter.Score method) (stages.text.modules.score_filter.ScoreFilter method) (stages.video.caption.caption_enhancement.CaptionEnhancementStage method) (stages.video.caption.caption_generation.CaptionGenerationStage method) (stages.video.caption.caption_preparation.CaptionPreparationStage method) (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage method) (stages.video.clipping.clip_extraction_stages.FixedStrideExtractorStage method) (stages.video.clipping.clip_frame_extraction.ClipFrameExtractionStage method) (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage method) (stages.video.clipping.video_frame_extraction.VideoFrameExtractionStage method) (stages.video.embedding.cosmos_embed1.CosmosEmbed1EmbeddingStage method) (stages.video.embedding.cosmos_embed1.CosmosEmbed1FrameCreationStage method) (stages.video.embedding.internvideo2.InternVideo2EmbeddingStage method) (stages.video.embedding.internvideo2.InternVideo2FrameCreationStage method) (stages.video.filtering.clip_aesthetic_filter.ClipAestheticFilterStage method) (stages.video.filtering.motion_filter.MotionFilterStage method) (stages.video.filtering.motion_filter.MotionVectorDecodeStage method) (stages.video.io.clip_writer.ClipWriterStage method) (stages.video.io.video_reader.VideoReaderStage method) (stages.video.preview.preview.PreviewStage method)

insert_chunk() (stages.deduplication.shuffle_utils.rapidsmpf_shuffler.BulkRapidsMPFShuffler method) insert_finished() (backends.experimental.ray_actor_pool.shuffle_adapter.ShuffleStageAdapter method) (stages.deduplication.exact.identification.ExactDuplicateIdentification method) (stages.deduplication.fuzzy.identify_duplicates.IdentifyDuplicatesStage method) (stages.deduplication.fuzzy.lsh.stage.LSHStage method) (stages.deduplication.shuffle_utils.rapidsmpf_shuffler.BulkRapidsMPFShuffler method) (stages.deduplication.shuffle_utils.stage.ShuffleStage method) INSTRUCTION_DATA_GUARD_MODEL_IDENTIFIER (in module stages.text.classifiers.aegis) InstructionDataGuardClassifier (class in stages.text.classifiers.aegis) InstructionDataGuardNet (class in stages.text.classifiers.aegis) intern_video_2_embedding (tasks.video.Clip attribute) intern_video_2_frames (tasks.video.Clip attribute) intern_video_2_text_match (tasks.video.Clip attribute) InternVideo2EmbeddingStage (class in stages.video.embedding.internvideo2) InternVideo2FrameCreationStage (class in stages.video.embedding.internvideo2) invert (stages.text.modules.score_filter.Filter attribute) (stages.text.modules.score_filter.ScoreFilter attribute) is_10_bit_color() (tasks.video.Video method) IS_ACTOR_STAGE (backends.experimental.utils.RayStageSpecKeys attribute) is_actor_stage() (in module backends.experimental.ray_data.utils) IS_FANOUT_STAGE (backends.experimental.utils.RayStageSpecKeys attribute) IS_LSH_STAGE (backends.experimental.utils.RayStageSpecKeys attribute) is_not_empty() (in module utils.file_utils) is_paragraph_indices_in_top_or_bottom_only() (in module stages.text.utils.text_utils) IS_RAFT_ACTOR (backends.experimental.utils.RayStageSpecKeys attribute) is_remote_url() (in module utils.client_utils) IS_SHUFFLE_STAGE (backends.experimental.utils.RayStageSpecKeys attribute) is_small_motion (stages.video.filtering.motion_vector_backend.MotionInfo attribute) items() (utils.performance_utils.StagePerfStats method) iterate() (stages.text.download.arxiv.iterator.ArxivIterator method) (stages.text.download.base.iterator.DocumentIterator method) (stages.text.download.common_crawl.warc_iterator.CommonCrawlWarcIterator method) (stages.text.download.wikipedia.iterator.WikipediaIterator method) iterator (stages.text.download.base.iterator.DocumentIterateStage attribute) (stages.text.download.base.stage.DocumentDownloadExtractStage attribute)

J

ja_stopwords (in module stages.text.download.html_extractors.utils.ja_stopwords) JsonEncoderCustom (class in utils.writer_utils) JsonlReader (class in stages.text.io.reader.jsonl)

JsonlReaderStage (class in stages.text.io.reader.jsonl) JsonlWriter (class in stages.text.io.writer.jsonl) JusTextExtractor (class in stages.text.download.html_extractors.justext)

K

keep_document() (stages.text.filters.code.AlphaFilter method) (stages.text.filters.code.GeneralCommentToCodeFilter method) (stages.text.filters.code.HTMLBoilerplateFilter method) (stages.text.filters.code.NumberOfLinesOfCodeFilter method) (stages.text.filters.code.PerExtensionFilter method) (stages.text.filters.code.PythonCommentToCodeFilter method) (stages.text.filters.code.TokenizerFertilityFilter method) (stages.text.filters.code.XMLHeaderFilter method) (stages.text.filters.doc_filter.DocumentFilter method) (stages.text.filters.fasttext_filter.FastTextLangId method) (stages.text.filters.fasttext_filter.FastTextQualityFilter method) (stages.text.filters.heuristic_filter.BoilerPlateStringFilter method) (stages.text.filters.heuristic_filter.BulletsFilter method) (stages.text.filters.heuristic_filter.CommonEnglishWordsFilter method) (stages.text.filters.heuristic_filter.EllipsisFilter method) (stages.text.filters.heuristic_filter.HistogramFilter method) (stages.text.filters.heuristic_filter.LongWordFilter method) (stages.text.filters.heuristic_filter.MeanWordLengthFilter method) (stages.text.filters.heuristic_filter.NonAlphaNumericFilter method) (stages.text.filters.heuristic_filter.NumbersFilter method) (stages.text.filters.heuristic_filter.ParenthesesFilter method) (stages.text.filters.heuristic_filter.PornographicUrlsFilter method) (stages.text.filters.heuristic_filter.PunctuationFilter method) (stages.text.filters.heuristic_filter.RepeatedLinesByCharFilter method) (stages.text.filters.heuristic_filter.RepeatedLinesFilter method) (stages.text.filters.heuristic_filter.RepeatedParagraphsByCharFilter method) (stages.text.filters.heuristic_filter.RepeatedParagraphsFilter method) (stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter method) (stages.text.filters.heuristic_filter.RepeatingTopNGramsFilter method) (stages.text.filters.heuristic_filter.SubstringFilter method) (stages.text.filters.heuristic_filter.SymbolsToWordsFilter method) (stages.text.filters.heuristic_filter.TokenCountFilter method) (stages.text.filters.heuristic_filter.UrlsFilter method) (stages.text.filters.heuristic_filter.WhiteSpaceFilter method) (stages.text.filters.heuristic_filter.WordCountFilter method) (stages.text.filters.heuristic_filter.WordsWithoutAlphabetsFilter method)

keep_raw_pred (stages.text.classifiers.aegis.AegisClassifier attribute) (stages.text.classifiers.aegis.PostProcessAegisResponsesStage attribute) kill_id_generator_actor() (in module stages.deduplication.id_generator) kmeans_init (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) kmeans_max_iter (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) kmeans_max_samples_per_batch (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) kmeans_n_init (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) kmeans_oversampling_factor (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) kmeans_random_state (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) kmeans_tol (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) KMeansReadFitWriteStage (class in stages.deduplication.semantic.kmeans) KMeansStage (class in stages.deduplication.semantic.kmeans)

L

L2_DIST_TO_CENT_COL (in module stages.deduplication.semantic.kmeans) lang (stages.audio.datasets.fleurs.create_initial_manifest.CreateInitialManifestFleursStage attribute) lang_detect() (in module stages.text.download.utils) language (stages.text.download.wikipedia.url_generation.WikipediaUrlGenerator attribute) last (utils.decoder_utils.FrameExtractionPolicy attribute) lcm_multiple() (stages.video.clipping.clip_frame_extraction.ClipFrameExtractionStage method) LegacySpeechStage (class in stages.audio.common) limit (stages.file_partitioning.FilePartitioningStage attribute) (stages.text.download.base.url_generation.URLGenerationStage attribute) (stages.text.download.common_crawl.url_generation.BaseCommonCrawlUrlGenerator attribute) limit_clips (stages.video.clipping.clip_extraction_stages.FixedStrideExtractorStage attribute) (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage attribute)

LineRemover (class in stages.text.modifiers.line_remover) load_cfg() (stages.text.models.tokenizer.TokenizerStage method) load_model() (stages.text.filters.fasttext_filter.FastTextLangId method) (stages.text.filters.fasttext_filter.FastTextQualityFilter method) load_tokenizer() (stages.text.filters.heuristic_filter.TokenCountFilter method) log_stats (stages.video.io.clip_writer.ClipWriterStage attribute) log_stats() (utils.performance_utils.StageTimer method) LongWordFilter (class in stages.text.filters.heuristic_filter) LSHActor (class in stages.deduplication.fuzzy.lsh.lsh) LSHStage (class in stages.deduplication.fuzzy.lsh.stage)

M

MainCommonCrawlUrlGenerator (class in stages.text.download.common_crawl.url_generation) make_named_temporary_file() (in module utils.operation_utils) make_pipeline_named_temporary_file() (in module utils.operation_utils) make_pipeline_temporary_dir() (in module utils.operation_utils) make_temporary_dir() (in module utils.operation_utils) MARKDOWN_BOLD_REGEX (in module stages.text.modifiers.markdown_remover) MARKDOWN_ITALIC_REGEX (in module stages.text.modifiers.markdown_remover) MARKDOWN_LINK_REGEX (in module stages.text.modifiers.markdown_remover) MARKDOWN_UNDERLINE_REGEX (in module stages.text.modifiers.markdown_remover) MarkdownRemover (class in stages.text.modifiers.markdown_remover) max_chars (stages.text.classifiers.aegis.AegisClassifier attribute) (stages.text.classifiers.aegis.FormatAegisPromptStage attribute) (stages.text.classifiers.aegis.InstructionDataGuardClassifier attribute) (stages.text.classifiers.base.DistributedDataClassifier attribute) (stages.text.classifiers.prompt_task_complexity.PromptTaskComplexityClassifier attribute) (stages.text.embedders.base.EmbeddingCreatorStage attribute) max_clip_duration (tasks.video.ClipStats attribute) max_iter (stages.deduplication.semantic.kmeans.KMeansStage attribute) max_length_mode (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage attribute) max_length_s (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage attribute) max_output_tokens (stages.video.caption.caption_enhancement.CaptionEnhancementStage attribute) (stages.video.caption.caption_generation.CaptionGenerationStage attribute) MAX_PIXELS (in module utils.windowing_utils) MAX_RATIO (in module utils.windowing_utils) max_samples_per_batch (stages.deduplication.semantic.kmeans.KMeansStage attribute) MAX_SEQ_LENGTH (in module stages.text.classifiers.aegis) (in module stages.text.classifiers.content_type) (in module stages.text.classifiers.domain) (in module stages.text.classifiers.fineweb_edu) (in module stages.text.classifiers.prompt_task_complexity) (in module stages.text.classifiers.quality) max_seq_length (stages.text.classifiers.base.DistributedDataClassifier attribute) (stages.text.embedders.base.EmbeddingCreatorStage attribute) max_workers (stages.video.io.clip_writer.ClipWriterStage attribute) MeanPooling (class in stages.text.classifiers.prompt_task_complexity) MeanWordLengthFilter (class in stages.text.filters.heuristic_filter) MEDIA_ALIASES (in module stages.text.download.wikipedia.extract) metadata (tasks.image.ImageObject attribute) (tasks.video.Video attribute) metadata_based() (stages.deduplication.semantic.ranking.RankingStrategy class method) metadata_fields (stages.deduplication.semantic.kmeans.KMeansStage attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) middle (utils.decoder_utils.FrameExtractionPolicy attribute) min_clip_length_s (stages.video.clipping.clip_extraction_stages.FixedStrideExtractorStage attribute) min_length_s (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage attribute) MIN_PIXELS (in module utils.windowing_utils) MinHash (class in stages.deduplication.fuzzy.minhash) minhash32() (stages.deduplication.fuzzy.minhash.GPUMinHash method) minhash64() (stages.deduplication.fuzzy.minhash.GPUMinHash method) minhash_field (stages.deduplication.fuzzy.lsh.stage.LSHStage attribute) minhash_to_bands() (stages.deduplication.fuzzy.lsh.lsh.LSHActor method) minhashes_per_band (stages.deduplication.fuzzy.lsh.stage.LSHStage attribute) MinHashStage (class in stages.deduplication.fuzzy.minhash) mode (stages.text.io.writer.base.BaseWriter attribute) model_batch_size (stages.video.caption.caption_enhancement.CaptionEnhancementStage attribute) model_check_or_download() (stages.text.filters.fasttext_filter.FastTextLangId method) (stages.text.filters.fasttext_filter.FastTextQualityFilter method) (stages.text.filters.heuristic_filter.TokenCountFilter method) model_dir (stages.image.embedders.clip_embedder.ImageEmbeddingStage attribute) (stages.image.filters.aesthetic_filter.ImageAestheticFilterStage attribute) (stages.image.filters.base.BaseFilterStage attribute) (stages.video.caption.caption_enhancement.CaptionEnhancementStage attribute) (stages.video.caption.caption_generation.CaptionGenerationStage attribute) (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage attribute) (stages.video.embedding.cosmos_embed1.CosmosEmbed1EmbeddingStage attribute) (stages.video.embedding.cosmos_embed1.CosmosEmbed1FrameCreationStage attribute) (stages.video.embedding.internvideo2.InternVideo2EmbeddingStage attribute) (stages.video.embedding.internvideo2.InternVideo2FrameCreationStage attribute) (stages.video.filtering.clip_aesthetic_filter.ClipAestheticFilterStage attribute) model_does_preprocess (stages.video.caption.caption_generation.CaptionGenerationStage attribute) (stages.video.caption.caption_preparation.CaptionPreparationStage attribute) model_identifier (stages.text.classifiers.base.DistributedDataClassifier attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) (stages.text.embedders.base.EmbeddingCreatorStage attribute) model_inference_batch_size (stages.image.embedders.clip_embedder.ImageEmbeddingStage attribute) (stages.image.filters.aesthetic_filter.ImageAestheticFilterStage attribute) (stages.image.filters.base.BaseFilterStage attribute) (stages.text.classifiers.aegis.AegisClassifier attribute) (stages.text.classifiers.aegis.InstructionDataGuardClassifier attribute) (stages.text.classifiers.base.DistributedDataClassifier attribute) (stages.text.classifiers.prompt_task_complexity.PromptTaskComplexityClassifier attribute) (stages.text.embedders.base.EmbeddingCreatorStage attribute) model_name (stages.audio.inference.asr_nemo.InferenceAsrNemoStage attribute) model_variant (stages.video.caption.caption_enhancement.CaptionEnhancementStage attribute) (stages.video.caption.caption_generation.CaptionGenerationStage attribute) (stages.video.caption.caption_preparation.CaptionPreparationStage attribute) ModelStage (class in stages.text.models.model) modifier_fn (stages.text.modules.modifier.Modify attribute) Modify (class in stages.text.modules.modifier) modify_document() (stages.text.modifiers.c4.BoilerPlateStringModifier method) (stages.text.modifiers.doc_modifier.DocumentModifier method) (stages.text.modifiers.fasttext.FastTextLabelModifier method) (stages.text.modifiers.line_remover.LineRemover method) (stages.text.modifiers.markdown_remover.MarkdownRemover method) (stages.text.modifiers.newline_normalizer.NewlineNormalizer method) (stages.text.modifiers.quotation_remover.QuotationRemover method) (stages.text.modifiers.slicer.Slicer method) (stages.text.modifiers.unicode_reformatter.UnicodeReformatter method) (stages.text.modifiers.url_remover.UrlRemover method) module backends backends.base backends.experimental backends.experimental.ray_actor_pool backends.experimental.ray_actor_pool.adapter backends.experimental.ray_actor_pool.executor backends.experimental.ray_actor_pool.raft_adapter backends.experimental.ray_actor_pool.shuffle_adapter backends.experimental.ray_actor_pool.utils backends.experimental.ray_data backends.experimental.ray_data.adapter backends.experimental.ray_data.executor backends.experimental.ray_data.utils backends.experimental.utils backends.internal backends.internal.raft backends.internal.raft.ray_comms backends.utils backends.xenna backends.xenna.adapter backends.xenna.executor pipeline pipeline.pipeline stages stages.audio stages.audio.common stages.audio.datasets stages.audio.datasets.file_utils stages.audio.datasets.fleurs stages.audio.datasets.fleurs.create_initial_manifest stages.audio.inference stages.audio.inference.asr_nemo stages.audio.io stages.audio.io.convert stages.audio.metrics stages.audio.metrics.get_wer stages.base stages.client_partitioning stages.deduplication stages.deduplication.exact stages.deduplication.exact.identification stages.deduplication.exact.workflow stages.deduplication.fuzzy stages.deduplication.fuzzy.buckets_to_edges stages.deduplication.fuzzy.connected_components stages.deduplication.fuzzy.identify_duplicates stages.deduplication.fuzzy.lsh stages.deduplication.fuzzy.lsh.lsh stages.deduplication.fuzzy.lsh.stage stages.deduplication.fuzzy.minhash stages.deduplication.fuzzy.utils stages.deduplication.fuzzy.workflow stages.deduplication.gpu_utils stages.deduplication.id_generator stages.deduplication.io_utils stages.deduplication.semantic stages.deduplication.semantic.identify_duplicates stages.deduplication.semantic.kmeans stages.deduplication.semantic.pairwise stages.deduplication.semantic.pairwise_io stages.deduplication.semantic.ranking stages.deduplication.semantic.utils stages.deduplication.semantic.workflow stages.deduplication.shuffle_utils stages.deduplication.shuffle_utils.rapidsmpf_shuffler stages.deduplication.shuffle_utils.stage stages.file_partitioning stages.function_decorators stages.image stages.image.deduplication stages.image.deduplication.removal stages.image.embedders stages.image.embedders.clip_embedder stages.image.filters stages.image.filters.aesthetic_filter stages.image.filters.base stages.image.filters.nsfw_filter stages.image.io stages.image.io.convert stages.image.io.image_reader stages.image.io.image_writer stages.resources stages.text stages.text.classifiers stages.text.classifiers.aegis stages.text.classifiers.aegis_utils stages.text.classifiers.base stages.text.classifiers.constants stages.text.classifiers.content_type stages.text.classifiers.domain stages.text.classifiers.fineweb_edu stages.text.classifiers.prompt_task_complexity stages.text.classifiers.quality stages.text.deduplication stages.text.deduplication.removal stages.text.deduplication.removal_workflow stages.text.deduplication.semantic stages.text.download stages.text.download.arxiv stages.text.download.arxiv.download stages.text.download.arxiv.extract stages.text.download.arxiv.iterator stages.text.download.arxiv.stage stages.text.download.arxiv.url_generation stages.text.download.base stages.text.download.base.download stages.text.download.base.extract stages.text.download.base.iterator stages.text.download.base.stage stages.text.download.base.url_generation stages.text.download.common_crawl stages.text.download.common_crawl.download stages.text.download.common_crawl.extract stages.text.download.common_crawl.stage stages.text.download.common_crawl.url_generation stages.text.download.common_crawl.warc_iterator stages.text.download.html_extractors stages.text.download.html_extractors.base stages.text.download.html_extractors.justext stages.text.download.html_extractors.resiliparse stages.text.download.html_extractors.trafilatura stages.text.download.html_extractors.utils stages.text.download.html_extractors.utils.ja_stopwords stages.text.download.html_extractors.utils.th_stopwords stages.text.download.html_extractors.utils.zh_stopwords stages.text.download.utils stages.text.download.wikipedia stages.text.download.wikipedia.download stages.text.download.wikipedia.extract stages.text.download.wikipedia.iterator stages.text.download.wikipedia.stage stages.text.download.wikipedia.url_generation stages.text.embedders stages.text.embedders.base stages.text.embedders.utils stages.text.filters stages.text.filters.code stages.text.filters.doc_filter stages.text.filters.fasttext_filter stages.text.filters.heuristic_filter stages.text.io stages.text.io.reader stages.text.io.reader.base stages.text.io.reader.jsonl stages.text.io.reader.parquet stages.text.io.writer stages.text.io.writer.base stages.text.io.writer.jsonl stages.text.io.writer.parquet stages.text.io.writer.utils stages.text.models stages.text.models.model stages.text.models.tokenizer stages.text.models.utils stages.text.modifiers stages.text.modifiers.c4 stages.text.modifiers.doc_modifier stages.text.modifiers.fasttext stages.text.modifiers.line_remover stages.text.modifiers.markdown_remover stages.text.modifiers.newline_normalizer stages.text.modifiers.quotation_remover stages.text.modifiers.slicer stages.text.modifiers.unicode_reformatter stages.text.modifiers.url_remover stages.text.modules stages.text.modules.add_id stages.text.modules.modifier stages.text.modules.score_filter stages.text.utils stages.text.utils.constants stages.text.utils.text_utils stages.video stages.video.caption stages.video.caption.caption_enhancement stages.video.caption.caption_generation stages.video.caption.caption_preparation stages.video.clipping stages.video.clipping.clip_extraction_stages stages.video.clipping.clip_frame_extraction stages.video.clipping.transnetv2_extraction stages.video.clipping.video_frame_extraction stages.video.embedding stages.video.embedding.cosmos_embed1 stages.video.embedding.internvideo2 stages.video.filtering stages.video.filtering.clip_aesthetic_filter stages.video.filtering.motion_filter stages.video.filtering.motion_vector_backend stages.video.io stages.video.io.clip_writer stages.video.io.video_reader stages.video.preview stages.video.preview.preview tasks tasks.audio_batch tasks.document tasks.file_group tasks.image tasks.tasks tasks.utils tasks.video utils utils.client_utils utils.column_utils utils.decoder_utils utils.file_utils utils.grouping utils.hf_download_utils utils.nvcodec_utils utils.operation_utils utils.performance_utils utils.storage_utils utils.windowing_utils utils.writer_utils

motion_filter_batch_size (stages.video.filtering.motion_filter.MotionFilterStage attribute) motion_score_global_mean (tasks.video.Clip attribute) motion_score_per_patch_min_256 (tasks.video.Clip attribute) motion_vectors_to_flowfield() (in module stages.video.filtering.motion_vector_backend) MotionFilterStage (class in stages.video.filtering.motion_filter) MotionInfo (class in stages.video.filtering.motion_vector_backend) MotionVectorDecodeStage (class in stages.video.filtering.motion_filter) MulticlassHead (class in stages.text.classifiers.prompt_task_complexity) MULTILINGUAL_DOMAIN_MODEL_IDENTIFIER (in module stages.text.classifiers.domain) MultilingualDomainClassifier (class in stages.text.classifiers.domain)

N

n_clusters (stages.deduplication.semantic.kmeans.KMeansStage attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) n_init (stages.deduplication.semantic.kmeans.KMeansStage attribute) name (stages.base.ProcessingStage property) (stages.text.filters.doc_filter.DocumentFilter property) (stages.text.modifiers.doc_modifier.DocumentModifier property) nb_streams_per_gpu (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage attribute) NewlineNormalizer (class in stages.text.modifiers.newline_normalizer) NewsCommonCrawlUrlGenerator (class in stages.text.download.common_crawl.url_generation) ngrams (stages.text.filters.doc_filter.DocumentFilter property) node_id (backends.base.NodeInfo attribute) NODE_TYPES (in module stages.text.utils.text_utils) NodeInfo (class in backends.base) NON_SPACED_LANGUAGES (stages.text.download.html_extractors.base.HTMLExtractorAlgorithm attribute) NonAlphaNumericFilter (class in stages.text.filters.heuristic_filter) normalize_embeddings_col_in_df() (stages.deduplication.semantic.kmeans.KMeansReadFitWriteStage static method) nsfw_score (tasks.image.ImageObject attribute) num_bands (stages.deduplication.fuzzy.lsh.stage.LSHStage attribute) num_clip_chunks (tasks.video.Video attribute) num_clips_per_chunk (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage attribute) num_cpus (stages.video.clipping.clip_frame_extraction.ClipFrameExtractionStage attribute) (stages.video.embedding.cosmos_embed1.CosmosEmbed1FrameCreationStage attribute) num_cpus_per_worker (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage attribute) (stages.video.filtering.motion_filter.MotionVectorDecodeStage attribute) (stages.video.preview.preview.PreviewStage attribute) num_filtered_by_aesthetic (tasks.video.ClipStats attribute) num_filtered_by_motion (tasks.video.ClipStats attribute) num_frames (tasks.video.VideoMetadata attribute) (utils.decoder_utils.VideoMetadata attribute)

num_gpus_per_worker (stages.image.embedders.clip_embedder.ImageEmbeddingStage attribute) (stages.image.filters.aesthetic_filter.ImageAestheticFilterStage attribute) (stages.image.filters.base.BaseFilterStage attribute) (stages.image.io.image_reader.ImageReaderStage attribute) (stages.video.embedding.internvideo2.InternVideo2EmbeddingStage attribute) (stages.video.filtering.clip_aesthetic_filter.ClipAestheticFilterStage attribute) (stages.video.filtering.motion_filter.MotionFilterStage attribute) num_items (tasks.audio_batch.AudioBatch property) (tasks.document.DocumentBatch property) (tasks.file_group.FileGroupTask property) (tasks.image.ImageBatch property) (tasks.tasks.Task property) (tasks.video.VideoTask property) num_items_processed (utils.performance_utils.StagePerfStats attribute) num_passed (tasks.video.ClipStats attribute) num_threads (stages.image.io.image_reader.ImageReaderStage attribute) num_total_clips (tasks.video.Video attribute) num_transcoded (tasks.video.ClipStats attribute) num_with_caption (tasks.video.ClipStats attribute) num_with_embeddings (tasks.video.ClipStats attribute) num_with_webp (tasks.video.ClipStats attribute) num_workers() (stages.base.ProcessingStage method) num_workers_per_node (stages.image.deduplication.removal.ImageDuplicatesRemovalStage attribute) num_workers_per_node() (stages.text.download.base.download.DocumentDownloader method) (stages.text.download.wikipedia.download.WikipediaDownloader method) NumberOfLinesOfCodeFilter (class in stages.text.filters.code) NumbersFilter (class in stages.text.filters.heuristic_filter) nvdecs (stages.resources.Resources attribute) nvencs (stages.resources.Resources attribute) NvVideoDecoder (class in utils.nvcodec_utils)

O

open() (utils.client_utils.FSPath method) OPENAI_CLIP_MEAN (in module utils.windowing_utils) OPENAI_CLIP_STD (in module utils.windowing_utils) OUTPUT_COLUMNS (in module stages.text.classifiers.prompt_task_complexity) output_columns() (stages.text.download.arxiv.extract.ArxivExtractor method) (stages.text.download.arxiv.iterator.ArxivIterator method) (stages.text.download.base.extract.DocumentExtractor method) (stages.text.download.base.iterator.DocumentIterator method) (stages.text.download.common_crawl.extract.CommonCrawlHTMLExtractor method) (stages.text.download.common_crawl.warc_iterator.CommonCrawlWarcIterator method) (stages.text.download.wikipedia.extract.WikipediaExtractor method) (stages.text.download.wikipedia.iterator.WikipediaIterator method) output_dir (stages.image.io.image_writer.ImageWriterStage attribute) output_fields (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) (stages.text.modules.modifier.Modify attribute) output_file_extension (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) output_filetype (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) output_hw (stages.video.clipping.video_frame_extraction.VideoFrameExtractionStage attribute) output_kwargs (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) output_mode (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) output_path (stages.deduplication.fuzzy.lsh.stage.LSHStage attribute) (stages.deduplication.semantic.identify_duplicates.IdentifyDuplicatesStage attribute) (stages.deduplication.semantic.kmeans.KMeansStage attribute) (stages.deduplication.semantic.pairwise.PairwiseStage attribute) (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) (stages.video.io.clip_writer.ClipWriterStage attribute) outputs() (stages.audio.inference.asr_nemo.InferenceAsrNemoStage method) (stages.base.CompositeStage method) (stages.base.ProcessingStage method) (stages.deduplication.exact.identification.ExactDuplicateIdentification method) (stages.deduplication.fuzzy.minhash.MinHashStage method) (stages.deduplication.semantic.pairwise_io.ClusterWiseFilePartitioningStage method) (stages.file_partitioning.FilePartitioningStage method) (stages.image.deduplication.removal.ImageDuplicatesRemovalStage method) (stages.image.embedders.clip_embedder.ImageEmbeddingStage method) (stages.image.filters.base.BaseFilterStage method) (stages.image.io.image_reader.ImageReaderStage method) (stages.image.io.image_writer.ImageWriterStage method) (stages.text.classifiers.aegis.AegisClassifier method) (stages.text.classifiers.aegis.AegisModelStage method) (stages.text.classifiers.aegis.FormatAegisPromptStage method) (stages.text.classifiers.aegis.InstructionDataGuardClassifier method) (stages.text.classifiers.aegis.PostProcessAegisResponsesStage method) (stages.text.classifiers.base.ClassifierModelStage method) (stages.text.classifiers.base.DistributedDataClassifier method) (stages.text.classifiers.fineweb_edu.FineWebModelStage method) (stages.text.classifiers.prompt_task_complexity.PromptTaskComplexityClassifier method) (stages.text.classifiers.prompt_task_complexity.PromptTaskComplexityModelStage method) (stages.text.download.base.download.DocumentDownloadStage method) (stages.text.download.base.extract.DocumentExtractStage method) (stages.text.download.base.iterator.DocumentIterateStage method) (stages.text.download.base.url_generation.URLGenerationStage method) (stages.text.embedders.base.EmbeddingModelStage method) (stages.text.io.reader.base.BaseReader method) (stages.text.io.writer.base.BaseWriter method) (stages.text.models.model.ModelStage method) (stages.text.models.tokenizer.TokenizerStage method) (stages.text.modules.add_id.AddId method) (stages.text.modules.modifier.Modify method) (stages.text.modules.score_filter.Filter method) (stages.text.modules.score_filter.Score method) (stages.text.modules.score_filter.ScoreFilter method) (stages.video.caption.caption_enhancement.CaptionEnhancementStage method) (stages.video.caption.caption_generation.CaptionGenerationStage method) (stages.video.caption.caption_preparation.CaptionPreparationStage method) (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage method) (stages.video.clipping.clip_extraction_stages.FixedStrideExtractorStage method) (stages.video.clipping.clip_frame_extraction.ClipFrameExtractionStage method) (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage method) (stages.video.clipping.video_frame_extraction.VideoFrameExtractionStage method) (stages.video.embedding.cosmos_embed1.CosmosEmbed1EmbeddingStage method) (stages.video.embedding.cosmos_embed1.CosmosEmbed1FrameCreationStage method) (stages.video.embedding.internvideo2.InternVideo2EmbeddingStage method) (stages.video.embedding.internvideo2.InternVideo2FrameCreationStage method) (stages.video.filtering.clip_aesthetic_filter.ClipAestheticFilterStage method) (stages.video.filtering.motion_filter.MotionFilterStage method) (stages.video.filtering.motion_filter.MotionVectorDecodeStage method) (stages.video.io.clip_writer.ClipWriterStage method) (stages.video.io.video_reader.VideoReaderStage method) (stages.video.preview.preview.PreviewStage method)

oversampling_factor (stages.deduplication.semantic.kmeans.KMeansStage attribute) overwrite (stages.text.modules.add_id.AddId attribute)

P

padding_side (stages.text.classifiers.base.DistributedDataClassifier attribute) (stages.text.embedders.base.EmbeddingCreatorStage attribute) pairwise() (in module utils.grouping) pairwise_batch_size (stages.deduplication.semantic.pairwise.PairwiseStage attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) pairwise_cosine_similarity_batched() (in module stages.deduplication.semantic.pairwise) PairwiseCosineSimilarityStage (class in stages.deduplication.semantic.pairwise) PairwiseStage (class in stages.deduplication.semantic.pairwise) pandas_select_columns() (in module utils.file_utils) paragraphs (stages.text.filters.doc_filter.DocumentFilter property) ParenthesesFilter (class in stages.text.filters.heuristic_filter) ParquetReader (class in stages.text.io.reader.parquet) ParquetReaderStage (class in stages.text.io.reader.parquet) ParquetWriter (class in stages.text.io.writer.parquet) parse_docstrings() (in module stages.text.utils.text_utils) path (stages.text.io.writer.base.BaseWriter attribute) per_patch_min_256 (stages.video.filtering.motion_vector_backend.MotionInfo attribute) per_patch_min_256_threshold (stages.video.filtering.motion_filter.MotionFilterStage attribute) PerExtensionFilter (class in stages.text.filters.code) perform_removal (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) pipeline module Pipeline (class in pipeline.pipeline) pipeline.pipeline module pixel_format (tasks.video.VideoMetadata attribute) (utils.decoder_utils.VideoMetadata attribute) policy_substrings (in module stages.text.utils.constants) populate_metadata() (tasks.video.Video method) PornographicUrlsFilter (class in stages.text.filters.heuristic_filter) PostProcessAegisResponsesStage (class in stages.text.classifiers.aegis) pred_column (stages.text.classifiers.aegis.AegisClassifier attribute) (stages.text.classifiers.aegis.InstructionDataGuardClassifier attribute) (stages.text.classifiers.aegis.PostProcessAegisResponsesStage attribute) (stages.text.classifiers.base.DistributedDataClassifier attribute) pred_text_key (stages.audio.inference.asr_nemo.InferenceAsrNemoStage attribute) (stages.audio.metrics.get_wer.GetPairwiseWerStage attribute) preprocess_dtype (stages.video.caption.caption_preparation.CaptionPreparationStage attribute) PreserveByValueStage (class in stages.audio.common) PRETRAINED_MODEL_NAME_OR_PATH (in module stages.text.classifiers.aegis) PreviewStage (class in stages.video.preview.preview) prob_column (stages.text.classifiers.aegis.InstructionDataGuardClassifier attribute) (stages.text.classifiers.base.DistributedDataClassifier attribute) process() (stages.audio.common.LegacySpeechStage method) (stages.audio.datasets.fleurs.create_initial_manifest.CreateInitialManifestFleursStage method) (stages.audio.inference.asr_nemo.InferenceAsrNemoStage method) (stages.audio.io.convert.AudioToDocumentStage method) (stages.base.CompositeStage method) (stages.base.ProcessingStage method) (stages.client_partitioning.ClientPartitioningStage method) (stages.deduplication.exact.identification.ExactDuplicateIdentification method) (stages.deduplication.fuzzy.buckets_to_edges.BucketsToEdgesStage method) (stages.deduplication.fuzzy.connected_components.ConnectedComponentsStage method) (stages.deduplication.fuzzy.identify_duplicates.IdentifyDuplicatesStage method) (stages.deduplication.fuzzy.lsh.stage.LSHStage method) (stages.deduplication.fuzzy.minhash.MinHashStage method) (stages.deduplication.semantic.identify_duplicates.IdentifyDuplicatesStage method) (stages.deduplication.semantic.kmeans.KMeansReadFitWriteStage method) (stages.deduplication.semantic.pairwise.PairwiseCosineSimilarityStage method) (stages.deduplication.semantic.pairwise_io.ClusterWiseFilePartitioningStage method) (stages.deduplication.shuffle_utils.stage.ShuffleStage method) (stages.file_partitioning.FilePartitioningStage method) (stages.image.deduplication.removal.ImageDuplicatesRemovalStage method) (stages.image.embedders.clip_embedder.ImageEmbeddingStage method) (stages.image.filters.aesthetic_filter.ImageAestheticFilterStage method) (stages.image.filters.base.BaseFilterStage method) (stages.image.filters.nsfw_filter.ImageNSFWFilterStage method) (stages.image.io.convert.ConvertImageBatchToDocumentBatchStage method) (stages.image.io.image_reader.ImageReaderStage method) (stages.image.io.image_writer.ImageWriterStage method) (stages.text.classifiers.aegis.FormatAegisPromptStage method) (stages.text.classifiers.aegis.PostProcessAegisResponsesStage method) (stages.text.deduplication.removal.TextDuplicatesRemovalStage method) (stages.text.download.base.download.DocumentDownloadStage method) (stages.text.download.base.extract.DocumentExtractStage method) (stages.text.download.base.iterator.DocumentIterateStage method) (stages.text.download.base.url_generation.URLGenerationStage method) (stages.text.io.reader.base.BaseReader method) (stages.text.io.writer.base.BaseWriter method) (stages.text.models.model.ModelStage method) (stages.text.models.tokenizer.TokenizerStage method) (stages.text.modules.add_id.AddId method) (stages.text.modules.modifier.Modify method) (stages.text.modules.score_filter.Filter method) (stages.text.modules.score_filter.Score method) (stages.text.modules.score_filter.ScoreFilter method) (stages.video.caption.caption_enhancement.CaptionEnhancementStage method) (stages.video.caption.caption_generation.CaptionGenerationStage method) (stages.video.caption.caption_preparation.CaptionPreparationStage method) (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage method) (stages.video.clipping.clip_extraction_stages.FixedStrideExtractorStage method) (stages.video.clipping.clip_frame_extraction.ClipFrameExtractionStage method) (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage method) (stages.video.clipping.video_frame_extraction.VideoFrameExtractionStage method) (stages.video.embedding.cosmos_embed1.CosmosEmbed1EmbeddingStage method) (stages.video.embedding.cosmos_embed1.CosmosEmbed1FrameCreationStage method) (stages.video.embedding.internvideo2.InternVideo2EmbeddingStage method) (stages.video.embedding.internvideo2.InternVideo2FrameCreationStage method) (stages.video.filtering.clip_aesthetic_filter.ClipAestheticFilterStage method) (stages.video.filtering.motion_filter.MotionFilterStage method) (stages.video.filtering.motion_filter.MotionVectorDecodeStage method) (stages.video.io.clip_writer.ClipWriterStage method) (stages.video.io.video_reader.VideoReaderStage method) (stages.video.preview.preview.PreviewStage method)

process_batch() (backends.base.BaseStageAdapter method) (stages.base.ProcessingStage method) (stages.deduplication.fuzzy.connected_components.ConnectedComponentsStage method) (stages.deduplication.semantic.identify_duplicates.IdentifyDuplicatesStage method) (stages.deduplication.semantic.kmeans.KMeansReadFitWriteStage method) process_data() (backends.xenna.adapter.XennaStageAdapter method) process_dataset() (backends.experimental.ray_data.adapter.RayDataStageAdapter method) process_dataset_entry() (stages.audio.common.GetAudioDurationStage method) (stages.audio.common.LegacySpeechStage method) (stages.audio.common.PreserveByValueStage method) (stages.audio.metrics.get_wer.GetPairwiseWerStage method) process_logits() (stages.text.classifiers.prompt_task_complexity.CustomDeberta method) process_model_output() (stages.text.classifiers.aegis.AegisModelStage method) (stages.text.classifiers.base.ClassifierModelStage method) (stages.text.classifiers.fineweb_edu.FineWebModelStage method) (stages.text.classifiers.prompt_task_complexity.PromptTaskComplexityModelStage method) (stages.text.embedders.base.EmbeddingModelStage method) (stages.text.models.model.ModelStage method) process_time (utils.performance_utils.StagePerfStats attribute) process_transcript() (stages.audio.datasets.fleurs.create_initial_manifest.CreateInitialManifestFleursStage method) processing_stage() (in module stages.function_decorators) ProcessingStage (class in stages.base) PROMPT_TASK_COMPLEXITY_MODEL_IDENTIFIER (in module stages.text.classifiers.prompt_task_complexity) prompt_text (stages.video.caption.caption_enhancement.CaptionEnhancementStage attribute) (stages.video.caption.caption_preparation.CaptionPreparationStage attribute) prompt_variant (stages.video.caption.caption_enhancement.CaptionEnhancementStage attribute) (stages.video.caption.caption_preparation.CaptionPreparationStage attribute) PromptTaskComplexityClassifier (class in stages.text.classifiers.prompt_task_complexity) PromptTaskComplexityModelStage (class in stages.text.classifiers.prompt_task_complexity) PunctuationFilter (class in stages.text.filters.heuristic_filter) pyncv_batch_size (stages.video.clipping.video_frame_extraction.VideoFrameExtractionStage attribute) PyNvcFrameExtractor (class in utils.nvcodec_utils) PythonCommentToCodeFilter (class in stages.text.filters.code)

Q

quality (stages.video.preview.preview.PreviewStage attribute) QUALITY_CLASSIFIER_MODEL_IDENTIFIER (in module stages.text.classifiers.quality)

QualityClassifier (class in stages.text.classifiers.quality) QuotationRemover (class in stages.text.modifiers.quotation_remover)

R

random() (stages.deduplication.semantic.ranking.RankingStrategy class method) random_seed (stages.deduplication.semantic.pairwise.PairwiseStage attribute) random_state (stages.deduplication.semantic.kmeans.KMeansStage attribute) rank_cluster() (stages.deduplication.semantic.ranking.RankingStrategy method) ranking_strategy (stages.deduplication.semantic.pairwise.PairwiseStage attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) RankingStrategy (class in stages.deduplication.semantic.ranking) raw_data_dir (stages.audio.datasets.fleurs.create_initial_manifest.CreateInitialManifestFleursStage attribute) raw_pred_column (stages.text.classifiers.aegis.AegisClassifier attribute) (stages.text.classifiers.aegis.PostProcessAegisResponsesStage attribute) ray_stage_spec() (stages.base.ProcessingStage method) (stages.deduplication.exact.identification.ExactDuplicateIdentification method) (stages.deduplication.fuzzy.connected_components.ConnectedComponentsStage method) (stages.deduplication.fuzzy.identify_duplicates.IdentifyDuplicatesStage method) (stages.deduplication.fuzzy.lsh.stage.LSHStage method) (stages.deduplication.semantic.kmeans.KMeansReadFitWriteStage method) (stages.deduplication.semantic.pairwise_io.ClusterWiseFilePartitioningStage method) (stages.deduplication.shuffle_utils.stage.ShuffleStage method) (stages.file_partitioning.FilePartitioningStage method) (stages.text.classifiers.aegis.PostProcessAegisResponsesStage method) (stages.text.download.base.url_generation.URLGenerationStage method) (stages.text.io.reader.base.BaseReader method) (stages.text.models.tokenizer.TokenizerStage method) (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage method) RayActorPoolExecutor (class in backends.experimental.ray_actor_pool.executor) RayActorPoolRAFTAdapter (class in backends.experimental.ray_actor_pool.raft_adapter) RayActorPoolStageAdapter (class in backends.experimental.ray_actor_pool.adapter) RayDataExecutor (class in backends.experimental.ray_data.executor) RayDataStageAdapter (class in backends.experimental.ray_data.adapter) RayStageSpecKeys (class in backends.experimental.utils) read_and_insert() (backends.experimental.ray_actor_pool.shuffle_adapter.ShuffleStageAdapter method) (stages.deduplication.exact.identification.ExactDuplicateIdentification method) (stages.deduplication.fuzzy.identify_duplicates.IdentifyDuplicatesStage method) (stages.deduplication.fuzzy.lsh.lsh.LSHActor method) (stages.deduplication.fuzzy.lsh.stage.LSHStage method) (stages.deduplication.shuffle_utils.rapidsmpf_shuffler.BulkRapidsMPFShuffler method) (stages.deduplication.shuffle_utils.stage.ShuffleStage method) read_batch() (stages.deduplication.shuffle_utils.rapidsmpf_shuffler.BulkRapidsMPFShuffler method) read_data() (stages.text.io.reader.base.BaseReader method) (stages.text.io.reader.jsonl.JsonlReaderStage method) (stages.text.io.reader.parquet.ParquetReaderStage method) read_jsonl() (stages.deduplication.io_utils.DeduplicationIO method) read_kwargs (stages.deduplication.fuzzy.lsh.stage.LSHStage attribute) (stages.deduplication.semantic.identify_duplicates.IdentifyDuplicatesStage attribute) (stages.deduplication.semantic.kmeans.KMeansStage attribute) (stages.deduplication.semantic.pairwise.PairwiseStage attribute) (stages.text.deduplication.removal.TextDuplicatesRemovalStage attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) (stages.text.io.reader.base.BaseReader attribute) (stages.text.io.reader.jsonl.JsonlReader attribute) (stages.text.io.reader.parquet.ParquetReader attribute)

read_minhash() (stages.deduplication.fuzzy.lsh.lsh.LSHActor method) read_parquet() (stages.deduplication.io_utils.DeduplicationIO method) read_video_cpu() (in module utils.windowing_utils) reader_config (tasks.file_group.FileGroupTask attribute) record_limit (stages.text.download.base.iterator.DocumentIterateStage attribute) (stages.text.download.base.stage.DocumentDownloadExtractStage attribute) reduction (stages.video.filtering.clip_aesthetic_filter.ClipAestheticFilterStage attribute) regex_alpha (in module stages.text.utils.constants) regex_alphanum (in module stages.text.utils.constants) regex_digit (in module stages.text.utils.constants) regex_hash (in module stages.text.utils.constants) regex_paren (in module stages.text.utils.constants) regex_url (in module stages.text.utils.constants) register_batch() (stages.deduplication.id_generator.IdGeneratorBase method) register_loguru_serializer() (in module backends.utils) reinit() (utils.performance_utils.StageTimer method) remainder_threshold (stages.video.caption.caption_preparation.CaptionPreparationStage attribute) removal_parquets_dir (stages.image.deduplication.removal.ImageDuplicatesRemovalStage attribute) remove_control_characters() (in module stages.text.download.utils) remove_image_data (stages.image.embedders.clip_embedder.ImageEmbeddingStage attribute) (stages.image.io.image_writer.ImageWriterStage attribute) remove_punctuation() (in module stages.text.utils.text_utils) RepeatedLinesByCharFilter (class in stages.text.filters.heuristic_filter) RepeatedLinesFilter (class in stages.text.filters.heuristic_filter) RepeatedParagraphsByCharFilter (class in stages.text.filters.heuristic_filter) RepeatedParagraphsFilter (class in stages.text.filters.heuristic_filter) RepeatingDuplicateNGramsFilter (class in stages.text.filters.heuristic_filter) RepeatingTopNGramsFilter (class in stages.text.filters.heuristic_filter) REQUEST_TIMEOUT (in module stages.text.download.wikipedia.url_generation) required_resources (backends.xenna.adapter.XennaStageAdapter property) requires_gpu (stages.resources.Resources property) reset() (utils.performance_utils.StagePerfStats method) ResiliparseExtractor (class in stages.text.download.html_extractors.resiliparse) Resolution (class in utils.decoder_utils) resolve_filename_column() (in module utils.column_utils) Resources (class in stages.resources) resources (stages.base.ProcessingStage property) rmm_pool_size (stages.deduplication.fuzzy.lsh.stage.LSHStage attribute) round_by_factor() (in module utils.windowing_utils) run() (pipeline.pipeline.Pipeline method) (stages.deduplication.exact.workflow.ExactDeduplicationWorkflow method) (stages.deduplication.fuzzy.workflow.FuzzyDeduplicationWorkflow method) (stages.deduplication.semantic.workflow.SemanticDeduplicationWorkflow method) (stages.text.deduplication.removal_workflow.TextDuplicatesRemovalWorkflow method) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow method)

S

sample_closest() (in module utils.decoder_utils) sampling_fps (stages.video.caption.caption_preparation.CaptionPreparationStage attribute) save_stream_position() (in module utils.decoder_utils) Score (class in stages.text.modules.score_filter) score_document() (stages.text.filters.code.AlphaFilter method) (stages.text.filters.code.GeneralCommentToCodeFilter method) (stages.text.filters.code.HTMLBoilerplateFilter method) (stages.text.filters.code.NumberOfLinesOfCodeFilter method) (stages.text.filters.code.PerExtensionFilter method) (stages.text.filters.code.PythonCommentToCodeFilter method) (stages.text.filters.code.TokenizerFertilityFilter method) (stages.text.filters.code.XMLHeaderFilter method) (stages.text.filters.doc_filter.DocumentFilter method) (stages.text.filters.fasttext_filter.FastTextLangId method) (stages.text.filters.fasttext_filter.FastTextQualityFilter method) (stages.text.filters.heuristic_filter.BoilerPlateStringFilter method) (stages.text.filters.heuristic_filter.BulletsFilter method) (stages.text.filters.heuristic_filter.CommonEnglishWordsFilter method) (stages.text.filters.heuristic_filter.EllipsisFilter method) (stages.text.filters.heuristic_filter.HistogramFilter method) (stages.text.filters.heuristic_filter.LongWordFilter method) (stages.text.filters.heuristic_filter.MeanWordLengthFilter method) (stages.text.filters.heuristic_filter.NonAlphaNumericFilter method) (stages.text.filters.heuristic_filter.NumbersFilter method) (stages.text.filters.heuristic_filter.ParenthesesFilter method) (stages.text.filters.heuristic_filter.PornographicUrlsFilter method) (stages.text.filters.heuristic_filter.PunctuationFilter method) (stages.text.filters.heuristic_filter.RepeatedLinesByCharFilter method) (stages.text.filters.heuristic_filter.RepeatedLinesFilter method) (stages.text.filters.heuristic_filter.RepeatedParagraphsByCharFilter method) (stages.text.filters.heuristic_filter.RepeatedParagraphsFilter method) (stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter method) (stages.text.filters.heuristic_filter.RepeatingTopNGramsFilter method) (stages.text.filters.heuristic_filter.SubstringFilter method) (stages.text.filters.heuristic_filter.SymbolsToWordsFilter method) (stages.text.filters.heuristic_filter.TokenCountFilter method) (stages.text.filters.heuristic_filter.UrlsFilter method) (stages.text.filters.heuristic_filter.WhiteSpaceFilter method) (stages.text.filters.heuristic_filter.WordCountFilter method) (stages.text.filters.heuristic_filter.WordsWithoutAlphabetsFilter method) score_field (stages.text.modules.score_filter.Score attribute) (stages.text.modules.score_filter.ScoreFilter attribute) score_fn (stages.text.modules.score_filter.Score attribute) score_only (stages.video.filtering.motion_filter.MotionFilterStage attribute) score_threshold (stages.image.filters.aesthetic_filter.ImageAestheticFilterStage attribute) (stages.image.filters.base.BaseFilterStage attribute) (stages.video.filtering.clip_aesthetic_filter.ClipAestheticFilterStage attribute) ScoreFilter (class in stages.text.modules.score_filter) SemanticDeduplicationWorkflow (class in stages.deduplication.semantic.workflow) sentences (stages.text.filters.doc_filter.DocumentFilter property) SEQ_ORDER_COLUMN (in module stages.text.models.utils) sequence (utils.decoder_utils.FrameExtractionPolicy attribute) set_root_unique_id() (backends.experimental.ray_actor_pool.raft_adapter.RayActorPoolRAFTAdapter method) setup() (backends.base.BaseStageAdapter method) (backends.experimental.ray_actor_pool.raft_adapter.RayActorPoolRAFTAdapter method) (backends.experimental.ray_actor_pool.shuffle_adapter.ShuffleStageAdapter method) (backends.xenna.adapter.XennaStageAdapter method) (stages.audio.inference.asr_nemo.InferenceAsrNemoStage method) (stages.base.ProcessingStage method) (stages.client_partitioning.ClientPartitioningStage method) (stages.deduplication.exact.identification.ExactDuplicateIdentification method) (stages.deduplication.fuzzy.connected_components.ConnectedComponentsStage method) (stages.deduplication.fuzzy.minhash.MinHashStage method) (stages.deduplication.semantic.kmeans.KMeansReadFitWriteStage method) (stages.deduplication.semantic.pairwise_io.ClusterWiseFilePartitioningStage method) (stages.image.deduplication.removal.ImageDuplicatesRemovalStage method) (stages.image.embedders.clip_embedder.ImageEmbeddingStage method) (stages.image.filters.aesthetic_filter.ImageAestheticFilterStage method) (stages.image.filters.base.BaseFilterStage method) (stages.image.filters.nsfw_filter.ImageNSFWFilterStage method) (stages.text.classifiers.aegis.PostProcessAegisResponsesStage method) (stages.text.embedders.base.EmbeddingModelStage method) (stages.text.io.reader.base.BaseReader method) (stages.text.models.model.ModelStage method) (stages.text.models.tokenizer.TokenizerStage method) (stages.text.modules.score_filter.Score method) (stages.text.modules.score_filter.ScoreFilter method) (stages.video.caption.caption_enhancement.CaptionEnhancementStage method) (stages.video.caption.caption_generation.CaptionGenerationStage method) (stages.video.caption.caption_preparation.CaptionPreparationStage method) (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage method) (stages.video.clipping.clip_frame_extraction.ClipFrameExtractionStage method) (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage method) (stages.video.clipping.video_frame_extraction.VideoFrameExtractionStage method) (stages.video.embedding.cosmos_embed1.CosmosEmbed1EmbeddingStage method) (stages.video.embedding.cosmos_embed1.CosmosEmbed1FrameCreationStage method) (stages.video.embedding.internvideo2.InternVideo2EmbeddingStage method) (stages.video.embedding.internvideo2.InternVideo2FrameCreationStage method) (stages.video.filtering.clip_aesthetic_filter.ClipAestheticFilterStage method) (stages.video.io.clip_writer.ClipWriterStage method) setup_on_node() (backends.base.BaseStageAdapter method) (backends.experimental.ray_actor_pool.adapter.RayActorPoolStageAdapter method) (backends.experimental.ray_actor_pool.raft_adapter.RayActorPoolRAFTAdapter method) (backends.experimental.ray_actor_pool.shuffle_adapter.ShuffleStageAdapter method) (backends.xenna.adapter.XennaStageAdapter method) (stages.audio.inference.asr_nemo.InferenceAsrNemoStage method) (stages.base.ProcessingStage method) (stages.image.embedders.clip_embedder.ImageEmbeddingStage method) (stages.image.filters.aesthetic_filter.ImageAestheticFilterStage method) (stages.image.filters.nsfw_filter.ImageNSFWFilterStage method) (stages.text.classifiers.aegis.PostProcessAegisResponsesStage method) (stages.text.models.model.ModelStage method) (stages.text.models.tokenizer.TokenizerStage method) (stages.text.modules.score_filter.Score method) (stages.text.modules.score_filter.ScoreFilter method) (stages.video.caption.caption_enhancement.CaptionEnhancementStage method) (stages.video.caption.caption_generation.CaptionGenerationStage method) (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage method) (stages.video.embedding.cosmos_embed1.CosmosEmbed1EmbeddingStage method) (stages.video.embedding.cosmos_embed1.CosmosEmbed1FrameCreationStage method) (stages.video.embedding.internvideo2.InternVideo2EmbeddingStage method) (stages.video.filtering.clip_aesthetic_filter.ClipAestheticFilterStage method) setup_root() (backends.experimental.ray_actor_pool.shuffle_adapter.ShuffleStageAdapter method) setup_worker() (backends.experimental.ray_actor_pool.shuffle_adapter.ShuffleStageAdapter method) (stages.deduplication.shuffle_utils.rapidsmpf_shuffler.BulkRapidsMPFShuffler method) ShuffleStage (class in stages.deduplication.shuffle_utils.stage) ShuffleStageAdapter (class in backends.experimental.ray_actor_pool.shuffle_adapter) sim_metric (stages.deduplication.semantic.pairwise.PairwiseStage attribute) size (tasks.video.VideoMetadata attribute) Slicer (class in stages.text.modifiers.slicer) smart_nframes() (in module utils.windowing_utils) smart_resize() (in module utils.windowing_utils) sort_by_length (stages.text.classifiers.aegis.AegisClassifier attribute) (stages.text.classifiers.aegis.InstructionDataGuardClassifier attribute) (stages.text.classifiers.base.DistributedDataClassifier attribute) (stages.text.classifiers.prompt_task_complexity.PromptTaskComplexityClassifier attribute) (stages.text.embedders.base.EmbeddingCreatorStage attribute) source_bytes (tasks.video.Video attribute) source_video (tasks.video.Clip attribute) span (tasks.video.Clip attribute) spill_memory_limit (stages.deduplication.fuzzy.lsh.stage.LSHStage attribute) split (stages.audio.datasets.fleurs.create_initial_manifest.CreateInitialManifestFleursStage attribute) split_by_chunk_size() (in module utils.grouping) split_into_n_chunks() (in module utils.grouping) split_video_into_windows() (in module utils.windowing_utils) stage2_prompt_text (stages.video.caption.caption_generation.CaptionGenerationStage attribute) stage_batch_size (backends.xenna.adapter.XennaStageAdapter property) stage_name (utils.performance_utils.StagePerfStats attribute) StageMeta (class in stages.base) StagePerfStats (class in utils.performance_utils) stages module stages.audio module stages.audio.common module stages.audio.datasets module stages.audio.datasets.file_utils module stages.audio.datasets.fleurs module stages.audio.datasets.fleurs.create_initial_manifest module stages.audio.inference module stages.audio.inference.asr_nemo module stages.audio.io module stages.audio.io.convert module stages.audio.metrics module stages.audio.metrics.get_wer module stages.base module stages.client_partitioning module stages.deduplication module stages.deduplication.exact module stages.deduplication.exact.identification module stages.deduplication.exact.workflow module stages.deduplication.fuzzy module stages.deduplication.fuzzy.buckets_to_edges module stages.deduplication.fuzzy.connected_components module stages.deduplication.fuzzy.identify_duplicates module stages.deduplication.fuzzy.lsh module stages.deduplication.fuzzy.lsh.lsh module stages.deduplication.fuzzy.lsh.stage module stages.deduplication.fuzzy.minhash module stages.deduplication.fuzzy.utils module stages.deduplication.fuzzy.workflow module stages.deduplication.gpu_utils module stages.deduplication.id_generator module stages.deduplication.io_utils module stages.deduplication.semantic module stages.deduplication.semantic.identify_duplicates module stages.deduplication.semantic.kmeans module stages.deduplication.semantic.pairwise module stages.deduplication.semantic.pairwise_io module stages.deduplication.semantic.ranking module stages.deduplication.semantic.utils module stages.deduplication.semantic.workflow module stages.deduplication.shuffle_utils module stages.deduplication.shuffle_utils.rapidsmpf_shuffler module stages.deduplication.shuffle_utils.stage module stages.file_partitioning module stages.function_decorators module stages.image module stages.image.deduplication module stages.image.deduplication.removal module stages.image.embedders module stages.image.embedders.clip_embedder module stages.image.filters module stages.image.filters.aesthetic_filter module

stages.image.filters.base module stages.image.filters.nsfw_filter module stages.image.io module stages.image.io.convert module stages.image.io.image_reader module stages.image.io.image_writer module stages.resources module stages.text module stages.text.classifiers module stages.text.classifiers.aegis module stages.text.classifiers.aegis_utils module stages.text.classifiers.base module stages.text.classifiers.constants module stages.text.classifiers.content_type module stages.text.classifiers.domain module stages.text.classifiers.fineweb_edu module stages.text.classifiers.prompt_task_complexity module stages.text.classifiers.quality module stages.text.deduplication module stages.text.deduplication.removal module stages.text.deduplication.removal_workflow module stages.text.deduplication.semantic module stages.text.download module stages.text.download.arxiv module stages.text.download.arxiv.download module stages.text.download.arxiv.extract module stages.text.download.arxiv.iterator module stages.text.download.arxiv.stage module stages.text.download.arxiv.url_generation module stages.text.download.base module stages.text.download.base.download module stages.text.download.base.extract module stages.text.download.base.iterator module stages.text.download.base.stage module stages.text.download.base.url_generation module stages.text.download.common_crawl module stages.text.download.common_crawl.download module stages.text.download.common_crawl.extract module stages.text.download.common_crawl.stage module stages.text.download.common_crawl.url_generation module stages.text.download.common_crawl.warc_iterator module stages.text.download.html_extractors module stages.text.download.html_extractors.base module stages.text.download.html_extractors.justext module stages.text.download.html_extractors.resiliparse module stages.text.download.html_extractors.trafilatura module stages.text.download.html_extractors.utils module stages.text.download.html_extractors.utils.ja_stopwords module stages.text.download.html_extractors.utils.th_stopwords module stages.text.download.html_extractors.utils.zh_stopwords module stages.text.download.utils module stages.text.download.wikipedia module stages.text.download.wikipedia.download module stages.text.download.wikipedia.extract module stages.text.download.wikipedia.iterator module stages.text.download.wikipedia.stage module stages.text.download.wikipedia.url_generation module stages.text.embedders module stages.text.embedders.base module stages.text.embedders.utils module stages.text.filters module stages.text.filters.code module stages.text.filters.doc_filter module stages.text.filters.fasttext_filter module stages.text.filters.heuristic_filter module stages.text.io module stages.text.io.reader module stages.text.io.reader.base module stages.text.io.reader.jsonl module stages.text.io.reader.parquet module stages.text.io.writer module stages.text.io.writer.base module stages.text.io.writer.jsonl module stages.text.io.writer.parquet module stages.text.io.writer.utils module stages.text.models module stages.text.models.model module stages.text.models.tokenizer module stages.text.models.utils module stages.text.modifiers module stages.text.modifiers.c4 module stages.text.modifiers.doc_modifier module stages.text.modifiers.fasttext module stages.text.modifiers.line_remover module stages.text.modifiers.markdown_remover module stages.text.modifiers.newline_normalizer module stages.text.modifiers.quotation_remover module stages.text.modifiers.slicer module stages.text.modifiers.unicode_reformatter module stages.text.modifiers.url_remover module stages.text.modules module stages.text.modules.add_id module stages.text.modules.modifier module stages.text.modules.score_filter module stages.text.utils module stages.text.utils.constants module stages.text.utils.text_utils module stages.video module stages.video.caption module stages.video.caption.caption_enhancement module stages.video.caption.caption_generation module stages.video.caption.caption_preparation module stages.video.clipping module stages.video.clipping.clip_extraction_stages module stages.video.clipping.clip_frame_extraction module stages.video.clipping.transnetv2_extraction module stages.video.clipping.video_frame_extraction module stages.video.embedding module stages.video.embedding.cosmos_embed1 module stages.video.embedding.internvideo2 module stages.video.filtering module stages.video.filtering.clip_aesthetic_filter module stages.video.filtering.motion_filter module stages.video.filtering.motion_vector_backend module stages.video.io module stages.video.io.clip_writer module stages.video.io.video_reader module stages.video.preview module stages.video.preview.preview module StageTimer (class in utils.performance_utils) start (utils.windowing_utils.WindowFrameInfo attribute) start_snapshot_str (stages.text.download.common_crawl.url_generation.BaseCommonCrawlUrlGenerator attribute) storage_options (stages.file_partitioning.FilePartitioningStage attribute) SubstringFilter (class in stages.text.filters.heuristic_filter) supports_batch_processing() (stages.base.ProcessingStage method) SymbolsToWordsFilter (class in stages.text.filters.heuristic_filter)

T

T (in module tasks.tasks) (in module utils.grouping) tar_safe_extract() (in module utils.file_utils) target_duration_ratio (stages.video.filtering.motion_filter.MotionVectorDecodeStage attribute) target_fps (stages.video.clipping.clip_frame_extraction.ClipFrameExtractionStage attribute) (stages.video.embedding.cosmos_embed1.CosmosEmbed1FrameCreationStage attribute) (stages.video.embedding.internvideo2.InternVideo2FrameCreationStage attribute) (stages.video.filtering.clip_aesthetic_filter.ClipAestheticFilterStage attribute) (stages.video.filtering.motion_filter.MotionVectorDecodeStage attribute) (stages.video.preview.preview.PreviewStage attribute) (utils.decoder_utils.FrameExtractionSignature attribute) target_height (stages.video.preview.preview.PreviewStage attribute) target_res (stages.video.clipping.clip_frame_extraction.ClipFrameExtractionStage attribute) Task (class in tasks.tasks) task_batch_size (stages.image.io.image_reader.ImageReaderStage attribute) task_id (tasks.tasks.Task attribute) task_type (stages.text.io.reader.jsonl.JsonlReader attribute) (stages.text.io.reader.parquet.ParquetReader attribute) TaskPerfUtils (class in tasks.utils) tasks module tasks.audio_batch module tasks.document module tasks.file_group module tasks.image module tasks.tasks module tasks.utils module tasks.video module teardown() (backends.base.BaseStageAdapter method) (backends.experimental.ray_actor_pool.raft_adapter.RayActorPoolRAFTAdapter method) (backends.experimental.ray_actor_pool.shuffle_adapter.ShuffleStageAdapter method) (stages.base.ProcessingStage method) (stages.deduplication.fuzzy.lsh.stage.LSHStage method) (stages.deduplication.shuffle_utils.stage.ShuffleStage method) (stages.text.models.model.ModelStage method)

text_field (stages.text.classifiers.aegis.AegisClassifier attribute) (stages.text.classifiers.aegis.FormatAegisPromptStage attribute) (stages.text.classifiers.aegis.InstructionDataGuardClassifier attribute) (stages.text.classifiers.base.DistributedDataClassifier attribute) (stages.text.classifiers.prompt_task_complexity.PromptTaskComplexityClassifier attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) (stages.text.embedders.base.EmbeddingCreatorStage attribute) (stages.text.modules.score_filter.Score attribute) (stages.text.modules.score_filter.ScoreFilter attribute) text_key (stages.audio.datasets.fleurs.create_initial_manifest.CreateInitialManifestFleursStage attribute) (stages.audio.metrics.get_wer.GetPairwiseWerStage attribute) TextDuplicatesRemovalStage (class in stages.text.deduplication.removal) TextDuplicatesRemovalWorkflow (class in stages.text.deduplication.removal_workflow) texts_to_verify (stages.video.embedding.cosmos_embed1.CosmosEmbed1EmbeddingStage attribute) (stages.video.embedding.internvideo2.InternVideo2EmbeddingStage attribute) TextSemanticDeduplicationWorkflow (class in stages.text.deduplication.semantic) th_stopwords (in module stages.text.download.html_extractors.utils.th_stopwords) THREE_OR_MORE_NEWLINES_REGEX (in module stages.text.modifiers.newline_normalizer) THREE_OR_MORE_WINDOWS_NEWLINES_REGEX (in module stages.text.modifiers.newline_normalizer) threshold (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage attribute) time_process() (utils.performance_utils.StageTimer method) TIn (in module stages.function_decorators) to_dict() (utils.performance_utils.StagePerfStats method) to_disk() (stages.deduplication.id_generator.IdGeneratorBase method) to_pandas() (tasks.document.DocumentBatch method) to_pyarrow() (tasks.document.DocumentBatch method) to_str() (utils.decoder_utils.FrameExtractionSignature method) TOKEN_LENGTH_COLUMN (in module stages.text.models.utils) TokenCountFilter (class in stages.text.filters.heuristic_filter) TOKENIZER_PADDING_SIDE (in module stages.text.classifiers.aegis) TokenizerFertilityFilter (class in stages.text.filters.code) TokenizerStage (class in stages.text.models.tokenizer) tol (stages.deduplication.semantic.kmeans.KMeansStage attribute) TORCH_DTYPE (in module stages.text.classifiers.aegis) total_clip_duration (tasks.video.ClipStats attribute) total_nparts (stages.deduplication.fuzzy.lsh.stage.LSHStage attribute) TOut (in module stages.function_decorators) TrafilaturaExtractor (class in stages.text.download.html_extractors.trafilatura) transcribe() (stages.audio.inference.asr_nemo.InferenceAsrNemoStage method) TransNetV2ClipExtractionStage (class in stages.video.clipping.transnetv2_extraction) try_decode_with_detected_encoding() (in module stages.text.download.utils)

U

UnicodeReformatter (class in stages.text.modifiers.unicode_reformatter) unsafe_categories (in module stages.text.classifiers.aegis_utils) upload_clips (stages.video.io.clip_writer.ClipWriterStage attribute) url_generator (stages.text.download.base.stage.DocumentDownloadExtractStage attribute) (stages.text.download.base.url_generation.URLGenerationStage attribute) url_limit (stages.text.download.base.stage.DocumentDownloadExtractStage attribute) URL_REGEX (in module stages.text.modifiers.url_remover) URLGenerationStage (class in stages.text.download.base.url_generation) URLGenerator (class in stages.text.download.base.url_generation) UrlRemover (class in stages.text.modifiers.url_remover) UrlsFilter (class in stages.text.filters.heuristic_filter) use_hwaccel (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage attribute) use_id_generator (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) use_input_bit_rate (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage attribute) utils module utils.client_utils module utils.column_utils module

utils.decoder_utils module utils.file_utils module utils.grouping module utils.hf_download_utils module utils.nvcodec_utils module utils.operation_utils module utils.performance_utils module utils.storage_utils module utils.windowing_utils module utils.writer_utils module uuid (tasks.video.Clip attribute)

V

valid_nccl_placements (backends.internal.raft.ray_comms.Comms attribute) validate() (tasks.audio_batch.AudioBatch method) (tasks.document.DocumentBatch method) (tasks.file_group.FileGroupTask method) (tasks.image.ImageBatch method) (tasks.tasks.Task method) (tasks.video.VideoTask method) validate_input() (stages.base.ProcessingStage method) validate_item() (tasks.audio_batch.AudioBatch method) variant (stages.video.embedding.cosmos_embed1.CosmosEmbed1EmbeddingStage attribute) (stages.video.embedding.cosmos_embed1.CosmosEmbed1FrameCreationStage attribute) verbose (stages.deduplication.semantic.identify_duplicates.IdentifyDuplicatesStage attribute) (stages.deduplication.semantic.kmeans.KMeansStage attribute) (stages.deduplication.semantic.pairwise.PairwiseStage attribute) (stages.image.deduplication.removal.ImageDuplicatesRemovalStage attribute) (stages.image.embedders.clip_embedder.ImageEmbeddingStage attribute) (stages.image.filters.aesthetic_filter.ImageAestheticFilterStage attribute) (stages.image.filters.base.BaseFilterStage attribute) (stages.image.io.image_reader.ImageReaderStage attribute) (stages.image.io.image_writer.ImageWriterStage attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) (stages.video.caption.caption_enhancement.CaptionEnhancementStage attribute) (stages.video.caption.caption_generation.CaptionGenerationStage attribute) (stages.video.caption.caption_preparation.CaptionPreparationStage attribute) (stages.video.clipping.clip_extraction_stages.ClipTranscodingStage attribute) (stages.video.clipping.clip_extraction_stages.FixedStrideExtractorStage attribute) (stages.video.clipping.clip_frame_extraction.ClipFrameExtractionStage attribute) (stages.video.clipping.transnetv2_extraction.TransNetV2ClipExtractionStage attribute) (stages.video.clipping.video_frame_extraction.VideoFrameExtractionStage attribute) (stages.video.embedding.cosmos_embed1.CosmosEmbed1EmbeddingStage attribute) (stages.video.embedding.cosmos_embed1.CosmosEmbed1FrameCreationStage attribute) (stages.video.embedding.internvideo2.InternVideo2EmbeddingStage attribute) (stages.video.embedding.internvideo2.InternVideo2FrameCreationStage attribute) (stages.video.filtering.clip_aesthetic_filter.ClipAestheticFilterStage attribute) (stages.video.filtering.motion_filter.MotionFilterStage attribute) (stages.video.filtering.motion_filter.MotionVectorDecodeStage attribute) (stages.video.io.clip_writer.ClipWriterStage attribute) (stages.video.io.video_reader.VideoReader attribute) (stages.video.io.video_reader.VideoReaderStage attribute) (stages.video.preview.preview.PreviewStage attribute)

Video (class in tasks.video) video_codec (tasks.video.VideoMetadata attribute) (utils.decoder_utils.VideoMetadata attribute) video_duration (utils.decoder_utils.VideoMetadata attribute) video_limit (stages.video.io.video_reader.VideoReader attribute) VIDEO_MAX_PIXELS (in module utils.windowing_utils) VIDEO_MIN_PIXELS (in module utils.windowing_utils) VIDEO_TOTAL_PIXELS (in module utils.windowing_utils) VideoBatchDecoder (class in utils.nvcodec_utils) VideoFrameExtractionStage (class in stages.video.clipping.video_frame_extraction) VideoMetadata (class in tasks.video) (class in utils.decoder_utils) VideoReader (class in stages.video.io.video_reader) VideoReaderStage (class in stages.video.io.video_reader) VideoResolutionTooSmallError VideoTask (class in tasks.video)

W

weakly_connected_components() (stages.deduplication.fuzzy.connected_components.ConnectedComponentsStage method) weight (tasks.video.Video property) weights_path (stages.image.filters.nsfw_filter.ImageNSFWFilterStage attribute) wer_key (stages.audio.metrics.get_wer.GetPairwiseWerStage attribute) which_to_keep (stages.deduplication.semantic.pairwise.PairwiseStage attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) white_space_list (in module stages.text.utils.constants) WhiteSpaceFilter (class in stages.text.filters.heuristic_filter) width (tasks.video.VideoMetadata attribute) (utils.decoder_utils.Resolution attribute) (utils.decoder_utils.VideoMetadata attribute) wikidumps_index_prefix (stages.text.download.wikipedia.url_generation.WikipediaUrlGenerator attribute) WikipediaDownloader (class in stages.text.download.wikipedia.download) WikipediaDownloadExtractStage (class in stages.text.download.wikipedia.stage) WikipediaExtractor (class in stages.text.download.wikipedia.extract) WikipediaIterator (class in stages.text.download.wikipedia.iterator) WikipediaUrlGenerator (class in stages.text.download.wikipedia.url_generation) WINDOW_MIN_FRAMES (in module utils.windowing_utils) window_size (stages.video.caption.caption_preparation.CaptionPreparationStage attribute) WindowFrameInfo (class in utils.windowing_utils) windows (tasks.video.Clip attribute) with_() (stages.base.CompositeStage method) (stages.base.ProcessingStage method)

WordCountFilter (class in stages.text.filters.heuristic_filter) WordsWithoutAlphabetsFilter (class in stages.text.filters.heuristic_filter) worker_id (backends.base.WorkerMetadata attribute) WorkerMetadata (class in backends.base) write_bytes() (in module utils.writer_utils) write_csv() (in module utils.writer_utils) write_data() (stages.text.io.writer.base.BaseWriter method) (stages.text.io.writer.jsonl.JsonlWriter method) (stages.text.io.writer.parquet.ParquetWriter method) write_id_generator_to_disk() (in module stages.deduplication.id_generator) write_json() (in module utils.writer_utils) write_kwargs (stages.deduplication.fuzzy.lsh.stage.LSHStage attribute) (stages.deduplication.semantic.identify_duplicates.IdentifyDuplicatesStage attribute) (stages.deduplication.semantic.kmeans.KMeansStage attribute) (stages.deduplication.semantic.pairwise.PairwiseStage attribute) (stages.text.deduplication.semantic.TextSemanticDeduplicationWorkflow attribute) (stages.text.io.writer.base.BaseWriter attribute) (stages.text.io.writer.jsonl.JsonlWriter attribute) (stages.text.io.writer.parquet.ParquetWriter attribute) write_parquet() (in module utils.writer_utils) (stages.deduplication.io_utils.DeduplicationIO method) write_table() (stages.deduplication.shuffle_utils.rapidsmpf_shuffler.BulkRapidsMPFShuffler method)

X

X (in module stages.base) xenna_stage_spec() (stages.base.ProcessingStage method) (stages.image.deduplication.removal.ImageDuplicatesRemovalStage method) (stages.text.download.base.download.DocumentDownloadStage method)

XennaExecutor (class in backends.xenna.executor) XennaStageAdapter (class in backends.xenna.adapter) XMLHeaderFilter (class in stages.text.filters.code)

Y

Y (in module stages.base) yield_next_batch() (stages.image.embedders.clip_embedder.ImageEmbeddingStage method) (stages.image.filters.base.BaseFilterStage method) (stages.text.models.model.ModelStage method)

Z

zh_stopwords (in module stages.text.download.html_extractors.utils.zh_stopwords)