nv_ingest.util package

Subpackages

• nv_ingest.util.converters package
  • Submodules
  • nv_ingest.util.converters.bytetools module
    • base64frombytes()
    • bytesfrombase64()
    • bytesfromhex()
    • hexfrombytes()
  • nv_ingest.util.converters.containers module
    • merge_dict()
  • nv_ingest.util.converters.datetools module
    • remove_tz()
    • validate_iso8601()
  • nv_ingest.util.converters.dftools module
    • MemoryFiles
      • open()
    • cudf_to_json()
    • cudf_to_pandas()
    • pandas_to_cudf()
  • nv_ingest.util.converters.formats module
    • ingest_json_results_to_blob()
  • nv_ingest.util.converters.type_mappings module
    • doc_type_to_content_type()
  • Module contents
• nv_ingest.util.detectors package
  • Submodules
  • nv_ingest.util.detectors.language module
  • Module contents
• nv_ingest.util.exception_handlers package
  • Submodules
  • nv_ingest.util.exception_handlers.converters module
    • datetools_exception_handler()
  • nv_ingest.util.exception_handlers.decorators module
    • CMNVIngestFailureContextManager
    • cm_ensure_payload_not_null()
    • cm_set_failure()
    • nv_ingest_node_failure_context_manager()
    • nv_ingest_source_failure_context_manager()
  • nv_ingest.util.exception_handlers.detectors module
    • langdetect_exception_handler()
  • nv_ingest.util.exception_handlers.pdf module
    • create_exception_tag()
    • pdfium_exception_handler()
  • nv_ingest.util.exception_handlers.schemas module
    • schema_exception_handler()
  • Module contents
• nv_ingest.util.flow_control package
  • Submodules
  • nv_ingest.util.flow_control.filter_by_task module
    • filter_by_task()
  • Module contents
    • filter_by_task()
• nv_ingest.util.image_processing package
  • Submodules
  • nv_ingest.util.image_processing.clustering module
    • boxes_are_close_or_overlap()
    • combine_groups_into_bboxes()
    • group_bounding_boxes()
    • remove_superset_bboxes()
  • nv_ingest.util.image_processing.table_and_chart module
    • assign_boxes()
    • build_markdown()
    • convert_paddle_response_to_psuedo_markdown()
    • display_markdown()
    • join_yolox_graphic_elements_and_paddle_output()
    • join_yolox_table_structure_and_paddle_output()
    • match_bboxes()
    • merge_text_in_cell()
    • process_yolox_graphic_elements()
    • remove_empty_row()
  • nv_ingest.util.image_processing.transforms module
    • base64_to_numpy()
    • check_numpy_image_size()
    • crop_image()
    • ensure_base64_is_png()
    • normalize_image()
    • numpy_to_base64()
    • pad_image()
    • scale_image_to_encoding_size()
  • Module contents
• nv_ingest.util.logging package
  • Submodules
  • nv_ingest.util.logging.configuration module
    • LogLevel
      • CRITICAL
      • DEBUG
      • ERROR
      • INFO
      • WARNING
    • configure_logging()
  • Module contents
• nv_ingest.util.message_brokers package
  • Subpackages
    • nv_ingest.util.message_brokers.kafka package
      • Module contents
    • nv_ingest.util.message_brokers.redis package
      • Submodules
      • nv_ingest.util.message_brokers.redis.redis_client module
      • Module contents
    • nv_ingest.util.message_brokers.simple_message_broker package
      • Submodules
      • nv_ingest.util.message_brokers.simple_message_broker.broker module
      • nv_ingest.util.message_brokers.simple_message_broker.ordered_message_queue module
      • nv_ingest.util.message_brokers.simple_message_broker.simple_client module
      • Module contents
  • Submodules
  • nv_ingest.util.message_brokers.client_base module
    • MessageBrokerClientBase
      • fetch_message()
      • get_client()
      • ping()
      • submit_message()
  • Module contents
• nv_ingest.util.modules package
  • Submodules
  • nv_ingest.util.modules.config_validator module
    • fetch_and_validate_module_config()
  • Module contents
• nv_ingest.util.morpheus package
  • Submodules
  • nv_ingest.util.morpheus.linear_module_source_stage_cpu module
    • LinearModuleSourceStageCPU
      • supported_execution_modes()
    • LinearModuleStageCPU
      • supported_execution_modes()
  • Module contents
• nv_ingest.util.multi_processing package
  • Submodules
  • nv_ingest.util.multi_processing.mp_pool_singleton module
    • ProcessWorkerPoolSingleton
      • close()
      • submit_task()
    • SimpleFuture
      • result()
  • Module contents
    • ProcessWorkerPoolSingleton
      • close()
      • submit_task()
• nv_ingest.util.nim package
  • Submodules
  • nv_ingest.util.nim.cached module
    • CachedModelInterface
      • format_input()
      • name()
      • parse_output()
      • prepare_data_for_inference()
      • process_inference_results()
  • nv_ingest.util.nim.decorators module
    • multiprocessing_cache()
  • nv_ingest.util.nim.deplot module
    • DeplotModelInterface
      • format_input()
      • name()
      • parse_output()
      • prepare_data_for_inference()
      • process_inference_results()
  • nv_ingest.util.nim.helpers module
    • ModelInterface
      • format_input()
      • name()
      • parse_output()
      • prepare_data_for_inference()
      • process_inference_results()
    • NimClient
      • close()
      • infer()
      • try_set_max_batch_size()
    • create_inference_client()
    • generate_url()
    • get_model_name()
    • get_version()
    • is_ready()
    • preprocess_image_for_paddle()
    • remove_url_endpoints()
  • nv_ingest.util.nim.nemoretriever_parse module
    • NemoRetrieverParseModelInterface
      • format_input()
      • name()
      • parse_output()
      • prepare_data_for_inference()
      • process_inference_results()
  • nv_ingest.util.nim.paddle module
    • PaddleOCRModelInterface
      • format_input()
      • name()
      • parse_output()
      • prepare_data_for_inference()
      • process_inference_results()
  • nv_ingest.util.nim.text_embedding module
    • EmbeddingModelInterface
      • format_input()
      • name()
      • parse_output()
      • prepare_data_for_inference()
      • process_inference_results()
  • nv_ingest.util.nim.vlm module
    • VLMModelInterface
      • format_input()
      • name()
      • parse_output()
      • prepare_data_for_inference()
      • process_inference_results()
  • nv_ingest.util.nim.yolox module
    • YoloxGraphicElementsModelInterface
      • name()
      • postprocess_annotations()
    • YoloxModelInterfaceBase
      • format_input()
      • parse_output()
      • postprocess_annotations()
      • prepare_data_for_inference()
      • process_inference_results()
      • transform_normalized_coordinates_to_original()
    • YoloxPageElementsModelInterface
      • name()
      • postprocess_annotations()
    • YoloxTableStructureModelInterface
      • name()
      • postprocess_annotations()
    • batched_overlaps()
    • bb_iou_array()
    • expand_boxes()
    • expand_chart_bboxes()
    • expand_table_bboxes()
    • find_boxes_inside()
    • find_matching_box_fast()
    • get_bbox_dict_yolox_graphic()
    • get_bbox_dict_yolox_table()
    • get_biggest_box()
    • get_weighted_box()
    • get_yolox_model_name()
    • match_with_title()
    • merge_boxes()
    • merge_labels()
    • postprocess_model_prediction()
    • postprocess_results()
    • prefilter_boxes()
    • resize_image()
    • weighted_boxes_fusion()
  • Module contents
• nv_ingest.util.pdf package
  • Submodules
  • nv_ingest.util.pdf.metadata_aggregators module
    • Base64Image
      • bbox
      • height
      • image
      • max_height
      • max_width
      • width
    • CroppedImageWithContent
      • bbox
      • content
      • content_format
      • image
      • max_height
      • max_width
      • type_string
    • LatexTable
      • bbox
      • latex
      • max_height
      • max_width
    • PDFMetadata
      • date_created
      • filename
      • keywords
      • last_modified
      • page_count
      • source_type
    • construct_image_metadata_from_base64()
    • construct_image_metadata_from_pdf_image()
    • construct_text_metadata()
    • extract_pdf_metadata()
  • nv_ingest.util.pdf.pdfium module
    • convert_bitmap_to_corrected_numpy()
    • convert_pdfium_position()
    • extract_forms_from_pdfium_page()
    • extract_image_like_objects_from_pdfium_page()
    • extract_merged_images_from_pdfium_page()
    • extract_merged_shapes_from_pdfium_page()
    • extract_nested_simple_images_from_pdfium_page()
    • extract_simple_images_from_pdfium_page()
    • extract_top_level_simple_images_from_pdfium_page()
    • pdfium_pages_to_numpy()
    • pdfium_try_get_bitmap_as_numpy()
  • Module contents
• nv_ingest.util.pipeline package
  • Submodules
  • nv_ingest.util.pipeline.logging module
    • get_log_level()
    • setup_logging()
  • nv_ingest.util.pipeline.pipeline_builders module
    • setup_ingestion_pipeline()
  • nv_ingest.util.pipeline.pipeline_runners module
    • PipelineCreationSchema
      • audio_function_id
      • audio_grpc_endpoint
      • audio_infer_protocol
      • embedding_nim_endpoint
      • embedding_nim_model_name
      • ingest_log_level
      • max_ingest_process_workers
      • message_client_host
      • message_client_port
      • message_client_type
      • model_config
      • mrc_ignore_numa_check
      • nemoretriever_parse_http_endpoint
      • nemoretriever_parse_infer_protocol
      • nemoretriever_parse_model_name
      • ngc_api_key
      • nvidia_build_api_key
      • otel_exporter_otlp_endpoint
      • paddle_http_endpoint
      • paddle_infer_protocol
      • redis_morpheus_task_queue
      • vlm_caption_endpoint
      • vlm_caption_model_name
      • yolox_graphic_elements_http_endpoint
      • yolox_graphic_elements_infer_protocol
      • yolox_http_endpoint
      • yolox_infer_protocol
      • yolox_table_structure_http_endpoint
      • yolox_table_structure_infer_protocol
    • is_port_in_use()
    • run_ingest_pipeline()
    • run_pipeline()
    • start_pipeline_subprocess()
    • subprocess_entrypoint()
  • nv_ingest.util.pipeline.stage_builders module
    • add_audio_extractor_stage()
    • add_chart_extractor_stage()
    • add_completed_job_counter_stage()
    • add_docx_extractor_stage()
    • add_embed_extractions_stage()
    • add_embedding_storage_stage()
    • add_image_caption_stage()
    • add_image_dedup_stage()
    • add_image_extractor_stage()
    • add_image_filter_stage()
    • add_image_storage_stage()
    • add_infographic_extractor_stage()
    • add_metadata_injector_stage()
    • add_otel_meter_stage()
    • add_otel_tracer_stage()
    • add_pdf_extractor_stage()
    • add_pptx_extractor_stage()
    • add_sink_stage()
    • add_source_stage()
    • add_submitted_job_counter_stage()
    • add_table_extractor_stage()
    • add_text_splitter_stage()
    • add_vdb_task_sink_stage()
    • get_audio_retrieval_service()
    • get_caption_classifier_service()
    • get_default_cpu_count()
    • get_message_provider_config()
    • get_nim_service()
    • validate_positive()
  • Module contents
    • add_chart_extractor_stage()
    • add_docx_extractor_stage()
    • add_embed_extractions_stage()
    • add_embedding_storage_stage()
    • add_image_caption_stage()
    • add_image_dedup_stage()
    • add_image_extractor_stage()
    • add_image_filter_stage()
    • add_image_storage_stage()
    • add_metadata_injector_stage()
    • add_pdf_extractor_stage()
    • add_pptx_extractor_stage()
    • add_sink_stage()
    • add_source_stage()
    • add_submitted_job_counter_stage()
    • add_table_extractor_stage()
    • add_text_splitter_stage()
    • add_vdb_task_sink_stage()
    • setup_ingestion_pipeline()
• nv_ingest.util.schema package
  • Submodules
  • nv_ingest.util.schema.schema_validator module
  • Module contents
• nv_ingest.util.telemetry package
  • Submodules
  • nv_ingest.util.telemetry.global_stats module
    • GlobalStats
      • max_jobs
      • stats
      • job_stats
      • append_job_stat()
      • get_all_stats()
      • get_instance()
      • get_job_stat()
      • get_stat()
      • increment_stat()
      • reset_all_stats()
      • set_stat()
  • Module contents
• nv_ingest.util.tracing package
  • Submodules
  • nv_ingest.util.tracing.latency module
    • ColorCodes
      • BLUE
      • GREEN
      • RED
      • RESET
      • YELLOW
    • colorize()
    • latency_logger()
  • nv_ingest.util.tracing.logging module
    • TaskResultStatus
      • FAILURE
      • SUCCESS
    • annotate_cm()
    • annotate_task_result()
  • nv_ingest.util.tracing.tagging module
    • traceable()
    • traceable_func()
  • Module contents
    • latency_logger()
    • traceable()

Module contents