Index

A

add_by_ci_id() (impresso_essentials.versioning.data_manifest.DataManifest method) add_by_title_year() (impresso_essentials.versioning.data_manifest.DataManifest method) add_count_list_by_title_year() (impresso_essentials.versioning.data_manifest.DataManifest method) add_counts() (impresso_essentials.versioning.data_statistics.DataStatistics method) add_media_source_metadata() (in module impresso_essentials.versioning.helpers) add_provider_to_s3_partition() (in module impresso_essentials.io.s3_add_provider) add_stats_to_mft() (in module impresso_essentials.versioning.compute_manifest)

agg() (in module impresso_essentials.versioning.aggregators) aggregate_stats_for_title() (impresso_essentials.versioning.data_manifest.DataManifest method) alias (impresso_essentials.utils.IssueDir attribute) aliases_to_process() (in module impresso_essentials.versioning.compute_manifest) alternative_read_text() (in module impresso_essentials.io.s3) AO (impresso_essentials.utils.SourceMedium attribute) append_to_notes() (impresso_essentials.versioning.data_manifest.DataManifest method)

B

build_bbox_json() (in module impresso_essentials.bbox_visualizer.json_builder)

bytes_to() (in module impresso_essentials.utils)

C

CAN_CONSOLIDATED (impresso_essentials.utils.DataStage attribute) CANONICAL (impresso_essentials.utils.DataStage attribute) canonical_path() (in module impresso_essentials.io.fs_utils) check_filenaming() (in module impresso_essentials.io.fs_utils) check_id() (in module impresso_essentials.io.fs_utils) chunk() (in module impresso_essentials.utils) (in module impresso_essentials.versioning.aggregators) CLASSIF_IMAGES (impresso_essentials.utils.DataStage attribute) compute() (impresso_essentials.versioning.data_manifest.DataManifest method) compute_statistics() (in module impresso_essentials.io.s3_set_timestamp) compute_stats_for_stage() (in module impresso_essentials.versioning.compute_manifest) compute_stats_in_can_consolidated_bag() (in module impresso_essentials.versioning.aggregators) compute_stats_in_canonical_bag() (in module impresso_essentials.versioning.aggregators) compute_stats_in_classif_img_bag() (in module impresso_essentials.versioning.aggregators) compute_stats_in_doc_emb_bag() (in module impresso_essentials.versioning.aggregators) compute_stats_in_entities_bag() (in module impresso_essentials.versioning.aggregators) compute_stats_in_img_emb_bag() (in module impresso_essentials.versioning.aggregators)

compute_stats_in_langid_ocrqa_bag() (in module impresso_essentials.versioning.aggregators) compute_stats_in_langident_bag() (in module impresso_essentials.versioning.aggregators) compute_stats_in_lingproc_bag() (in module impresso_essentials.versioning.aggregators) compute_stats_in_ocrqa_bag() (in module impresso_essentials.versioning.aggregators) compute_stats_in_rebuilt_bag() (in module impresso_essentials.versioning.aggregators) compute_stats_in_solr_text_ing_bag() (in module impresso_essentials.versioning.aggregators) compute_stats_in_text_reuse_passage_bag() (in module impresso_essentials.versioning.aggregators) compute_stats_in_topics_bag() (in module impresso_essentials.versioning.aggregators) construct_dest_key() (in module impresso_essentials.io.s3_add_provider) count_keys (impresso_essentials.versioning.data_statistics.DataStatistics attribute) (impresso_essentials.versioning.data_statistics.MediaStatistics attribute) counts (impresso_essentials.versioning.data_statistics.DataStatistics attribute) (impresso_essentials.versioning.data_statistics.MediaStatistics attribute) counts_for_canonical_issue() (in module impresso_essentials.versioning.aggregators) counts_for_rebuilt() (in module impresso_essentials.versioning.aggregators) create_image_url() (in module impresso_essentials.bbox_visualizer.get_bbox) create_manifest() (in module impresso_essentials.versioning.compute_manifest) create_s3_path() (in module impresso_essentials.bbox_visualizer.get_bbox)

D

DataManifest (class in impresso_essentials.versioning.data_manifest) DataStage (class in impresso_essentials.utils) DataStatistics (class in impresso_essentials.versioning.data_statistics) date (impresso_essentials.utils.IssueDir attribute)

define_update_info_for_title() (impresso_essentials.versioning.data_manifest.DataManifest method) delete_versioned_keys() (in module impresso_essentials.io.s3_delete) disable_interrupts() (in module impresso_essentials.io.s3_set_timestamp) (in module impresso_essentials.utils)

E

EC (impresso_essentials.utils.SourceType attribute) edition (impresso_essentials.utils.IssueDir attribute) element (impresso_essentials.versioning.data_statistics.DataStatistics attribute) (impresso_essentials.versioning.data_statistics.MediaStatistics attribute) EMB_DOCS (impresso_essentials.utils.DataStage attribute) EMB_ENTITIES (impresso_essentials.utils.DataStage attribute)

EMB_IMAGES (impresso_essentials.utils.DataStage attribute) EMB_PARAGRAPHS (impresso_essentials.utils.DataStage attribute) EMB_SENTS (impresso_essentials.utils.DataStage attribute) EMB_WORDS (impresso_essentials.utils.DataStage attribute) ENTITIES (impresso_essentials.utils.DataStage attribute) extract_provider_alias_key() (in module impresso_essentials.io.s3) extract_version() (in module impresso_essentials.versioning.helpers)

F

fetch_files() (in module impresso_essentials.io.s3) filter_new_or_modified_media() (in module impresso_essentials.versioning.helpers) finalize() (in module impresso_essentials.versioning.aggregators)

find_s3_data_manifest_path() (in module impresso_essentials.versioning.helpers) fixed_s3fs_glob() (in module impresso_essentials.io.s3) freq() (in module impresso_essentials.versioning.aggregators)

G

generate_media_dict() (impresso_essentials.versioning.data_manifest.DataManifest method) get_alias_from_path() (in module impresso_essentials.io.s3_add_provider) get_base_url() (in module impresso_essentials.bbox_visualizer.get_bbox) get_bucket() (in module impresso_essentials.io.s3) get_ci_bounding_boxes() (in module impresso_essentials.bbox_visualizer.get_bbox) get_ci_type() (in module impresso_essentials.bbox_visualizer.get_bbox) get_count_keys() (impresso_essentials.versioning.data_manifest.DataManifest method) get_files_to_consider() (in module impresso_essentials.versioning.compute_manifest) get_issue_bounding_boxes() (in module impresso_essentials.bbox_visualizer.get_bbox) get_issueshortpath() (in module impresso_essentials.io.fs_utils) get_last_timestamp() (in module impresso_essentials.io.s3_set_timestamp) get_list_intersection() (in module impresso_essentials.utils) get_media_item_years() (in module impresso_essentials.versioning.helpers)

get_media_titles() (in module impresso_essentials.versioning.helpers) get_or_create_bucket() (in module impresso_essentials.io.s3) get_page_bounding_boxes() (in module impresso_essentials.bbox_visualizer.get_bbox) get_pkg_resource() (in module impresso_essentials.utils) get_provider_for_alias() (in module impresso_essentials.utils) get_s3_client() (in module impresso_essentials.io.s3) (in module impresso_essentials.io.s3_set_timestamp) get_s3_object_size() (in module impresso_essentials.io.s3) get_s3_resource() (in module impresso_essentials.io.s3) get_src_info_for_alias() (in module impresso_essentials.utils) get_storage_options() (in module impresso_essentials.io.s3) glob_with_size() (in module impresso_essentials.io.fs_utils) granularity (impresso_essentials.versioning.data_statistics.DataStatistics attribute) (impresso_essentials.versioning.data_statistics.MediaStatistics attribute)

H

has_title_year_key() (impresso_essentials.versioning.data_manifest.DataManifest method) has_value() (impresso_essentials.utils.DataStage class method) (impresso_essentials.utils.SourceMedium class method) (impresso_essentials.utils.SourceType class method)

I

id_to_issuedir() (in module impresso_essentials.utils) impresso_essentials.bbox_visualizer.get_bbox module impresso_essentials.bbox_visualizer.json_builder module impresso_essentials.io.fs_utils module impresso_essentials.io.s3 module impresso_essentials.io.s3_add_provider module impresso_essentials.io.s3_delete module impresso_essentials.io.s3_set_timestamp module impresso_essentials.text_utils module impresso_essentials.utils module

impresso_essentials.versioning.aggregators module impresso_essentials.versioning.compute_manifest module impresso_essentials.versioning.data_manifest module impresso_essentials.versioning.data_statistics module impresso_essentials.versioning.helpers module increment_version() (in module impresso_essentials.versioning.helpers) init_counts() (impresso_essentials.versioning.data_statistics.DataStatistics method) init_logger() (in module impresso_essentials.utils) init_media_info() (in module impresso_essentials.versioning.helpers) insert_whitespace() (in module impresso_essentials.text_utils) is_stopword_or_all_stopwords() (in module impresso_essentials.text_utils) IssueDir (class in impresso_essentials.utils)

L

LANGIDENT (impresso_essentials.utils.DataStage attribute) LANGIDENT_OCRQA (impresso_essentials.utils.DataStage attribute) LINGPROC (impresso_essentials.utils.DataStage attribute) list_canonical_files() (in module impresso_essentials.io.s3)

list_local_directories() (in module impresso_essentials.io.fs_utils) list_media_titles() (in module impresso_essentials.io.s3) list_providers_and_aliases() (in module impresso_essentials.io.s3) list_s3_directories() (in module impresso_essentials.io.s3) log_src_medium_mismatch() (in module impresso_essentials.versioning.aggregators)

M

main() (in module impresso_essentials.io.s3_add_provider) (in module impresso_essentials.io.s3_delete) (in module impresso_essentials.io.s3_set_timestamp) (in module impresso_essentials.versioning.compute_manifest) manifest_summary() (in module impresso_essentials.versioning.helpers) media_list_from_mft_json() (in module impresso_essentials.versioning.helpers) MediaStatistics (class in impresso_essentials.versioning.data_statistics) MG (impresso_essentials.utils.SourceType attribute) module impresso_essentials.bbox_visualizer.get_bbox impresso_essentials.bbox_visualizer.json_builder impresso_essentials.io.fs_utils impresso_essentials.io.s3 impresso_essentials.io.s3_add_provider impresso_essentials.io.s3_delete impresso_essentials.io.s3_set_timestamp impresso_essentials.text_utils impresso_essentials.utils impresso_essentials.versioning.aggregators impresso_essentials.versioning.compute_manifest impresso_essentials.versioning.data_manifest impresso_essentials.versioning.data_statistics impresso_essentials.versioning.helpers

MYSQL_CIS (impresso_essentials.utils.DataStage attribute)

N

new_media() (impresso_essentials.versioning.data_manifest.DataManifest method) NEWS_AGENCIES (impresso_essentials.utils.DataStage attribute)

normalize_text() (in module impresso_essentials.text_utils) NP (impresso_essentials.utils.SourceType attribute)

O

OCRQA (impresso_essentials.utils.DataStage attribute)

output_mft_s3_path (impresso_essentials.versioning.data_manifest.DataManifest property) overall_stats() (impresso_essentials.versioning.data_manifest.DataManifest method)

P

parse_canonical_filename() (in module impresso_essentials.io.fs_utils) parse_json() (in module impresso_essentials.io.fs_utils) partitioner() (in module impresso_essentials.utils) PASSIM (impresso_essentials.utils.DataStage attribute) path (impresso_essentials.utils.IssueDir attribute) possible_count_keys (impresso_essentials.versioning.data_statistics.MediaStatistics attribute) pretty_print() (impresso_essentials.versioning.data_statistics.DataStatistics method) (impresso_essentials.versioning.data_statistics.MediaStatistics method)

process_altogether() (in module impresso_essentials.versioning.compute_manifest) process_by_title() (in module impresso_essentials.versioning.compute_manifest) provider (impresso_essentials.utils.IssueDir attribute) provider_in_path() (in module impresso_essentials.io.s3) provider_level_stats() (impresso_essentials.versioning.data_manifest.DataManifest method) PT (impresso_essentials.utils.SourceMedium attribute)

R

RB (impresso_essentials.utils.SourceType attribute) read_jsonlines() (in module impresso_essentials.io.s3) read_manifest_from_s3() (in module impresso_essentials.versioning.helpers) read_manifest_from_s3_path() (in module impresso_essentials.versioning.helpers) read_s3_issues() (in module impresso_essentials.io.s3) readtext_jsonlines() (in module impresso_essentials.io.s3) REBUILT (impresso_essentials.utils.DataStage attribute)

remove_corrupted_files() (in module impresso_essentials.versioning.compute_manifest) remove_media_in_manifest() (in module impresso_essentials.versioning.helpers) replace_by_ci_id() (impresso_essentials.versioning.data_manifest.DataManifest method) replace_by_title_year() (impresso_essentials.versioning.data_manifest.DataManifest method) report_missing_metadata() (in module impresso_essentials.io.s3_set_timestamp) report_missing_metadata_dirs() (in module impresso_essentials.io.s3_set_timestamp) RM (impresso_essentials.utils.SourceType attribute) RS (impresso_essentials.utils.SourceType attribute)

S

s3_glob_with_size() (in module impresso_essentials.io.s3) s3_iter_bucket() (in module impresso_essentials.io.s3) same_counts() (impresso_essentials.versioning.data_statistics.DataStatistics method) (impresso_essentials.versioning.data_statistics.MediaStatistics method) search_text() (in module impresso_essentials.text_utils) segment_and_trim_sentences() (in module impresso_essentials.text_utils) SOLR_TEXT (impresso_essentials.utils.DataStage attribute)

sort_media_list_years_and_titles() (in module impresso_essentials.versioning.helpers) SourceMedium (class in impresso_essentials.utils) SourceType (class in impresso_essentials.utils) stage (impresso_essentials.versioning.data_statistics.DataStatistics attribute) (impresso_essentials.versioning.data_statistics.MediaStatistics attribute) stage_extra_keys (impresso_essentials.versioning.data_statistics.MediaStatistics attribute) stop() (impresso_essentials.utils.Timer method)

T

TEXT_REUSE (impresso_essentials.utils.DataStage attribute) tick() (impresso_essentials.utils.Timer method) Timer (class in impresso_essentials.utils) timestamp() (in module impresso_essentials.utils)

title_level_stats() (impresso_essentials.versioning.data_manifest.DataManifest method) tokenise() (in module impresso_essentials.text_utils) TOPICS (impresso_essentials.utils.DataStage attribute) TPS (impresso_essentials.utils.SourceMedium attribute)

U

update_media_stats() (impresso_essentials.versioning.data_manifest.DataManifest method) update_metadata_for_prefix() (in module impresso_essentials.io.s3_set_timestamp) update_metadata_if_needed() (in module impresso_essentials.io.s3_set_timestamp)

upload_to_s3() (in module impresso_essentials.io.s3) user_confirmation() (in module impresso_essentials.utils) user_question() (in module impresso_essentials.utils)

V

validate_against_schema() (in module impresso_essentials.utils) validate_and_export_manifest() (impresso_essentials.versioning.data_manifest.DataManifest method) validate_config() (in module impresso_essentials.versioning.compute_manifest) validate_granularity() (in module impresso_essentials.utils)

validate_source() (in module impresso_essentials.utils) validate_stage() (in module impresso_essentials.utils) validate_version() (in module impresso_essentials.versioning.helpers) version_as_list() (in module impresso_essentials.versioning.helpers)