swh.export package#

Subpackages#

• swh.export.exporters package
  • Submodules
    • swh.export.exporters.edges module
    • swh.export.exporters.orc module
  • Module contents
• swh.export.test namespace
  • Submodules
    • swh.export.test.test_edges module
    • swh.export.test.test_journal_processor module
    • swh.export.test.test_orc module
    • swh.export.test.test_utils module

Submodules#

• swh.export.athena module
  • create_database()
  • drop_table()
  • create_table()
  • repair_table()
  • query()
  • create_tables()
  • human_size()
  • run_query_get_results()
  • generate_subdataset()
• swh.export.cli module
  • run_export_graph()
• swh.export.exporter module
  • Exporter
    • Exporter.process_object()
    • Exporter.get_unique_file_id()
  • ExporterDispatch
    • ExporterDispatch.process_object()
• swh.export.fullnames module
  • process_fullnames()
• swh.export.journalprocessor module
  • FULLNAME_SIZE_LIMIT
  • JournalClientOffsetRanges
    • JournalClientOffsetRanges.subscribe()
    • JournalClientOffsetRanges.unsubscribe()
    • JournalClientOffsetRanges.process()
    • JournalClientOffsetRanges.handle_offset()
    • JournalClientOffsetRanges.deserialize_message()
    • JournalClientOffsetRanges.handle_messages()
  • ParallelJournalProcessor
    • ParallelJournalProcessor.get_offsets()
    • ParallelJournalProcessor.run()
    • ParallelJournalProcessor.progress_worker()
    • ParallelJournalProcessor.export_worker()
  • JournalProcessorWorker
    • JournalProcessorWorker.get_node_set_for_object()
    • JournalProcessorWorker.run()
    • JournalProcessorWorker.process_messages()
    • JournalProcessorWorker.process_message()
• swh.export.luigi module
  • Luigi tasks
    • File layout
    • Running all on staging
  • ObjectType
    • ObjectType.origin
    • ObjectType.origin_visit
    • ObjectType.origin_visit_status
    • ObjectType.snapshot
    • ObjectType.release
    • ObjectType.revision
    • ObjectType.directory
    • ObjectType.content
    • ObjectType.skipped_content
  • Format
    • Format.edges
    • Format.orc
  • merge_lists()
  • PathParameter
    • PathParameter.parse()
  • S3PathParameter
    • S3PathParameter.normalize()
  • FractionalFloatParameter
    • FractionalFloatParameter.parse()
  • stamps_paths()
  • get_masked_swhids()
  • StartExport
    • StartExport.config_file
    • StartExport.local_export_path
    • StartExport.local_sensitive_export_path
    • StartExport.export_id
    • StartExport.margin
    • StartExport.object_types
    • StartExport.output()
    • StartExport.complete()
    • StartExport.run()
  • ExportTopic
    • ExportTopic.config_file
    • ExportTopic.local_export_path
    • ExportTopic.local_sensitive_export_path
    • ExportTopic.export_id
    • ExportTopic.formats
    • ExportTopic.processes
    • ExportTopic.margin
    • ExportTopic.object_types
    • ExportTopic.requires()
    • ExportTopic.output()
    • ExportTopic.run()
  • ExportPersonsTable
    • ExportPersonsTable.config_file
    • ExportPersonsTable.local_export_path
    • ExportPersonsTable.local_sensitive_export_path
    • ExportPersonsTable.export_id
    • ExportPersonsTable.formats
    • ExportPersonsTable.processes
    • ExportPersonsTable.margin
    • ExportPersonsTable.object_types
    • ExportPersonsTable.requires()
    • ExportPersonsTable.output()
    • ExportPersonsTable.run()
  • ExportGraph
    • ExportGraph.config_file
    • ExportGraph.local_export_path
    • ExportGraph.local_sensitive_export_path
    • ExportGraph.export_id
    • ExportGraph.formats
    • ExportGraph.processes
    • ExportGraph.margin
    • ExportGraph.object_types
    • ExportGraph.export_name
    • ExportGraph.output()
    • ExportGraph.complete()
    • ExportGraph.get_export_id()
    • ExportGraph.requires()
    • ExportGraph.run()
  • UploadExportToS3
    • UploadExportToS3.local_export_path
    • UploadExportToS3.formats
    • UploadExportToS3.object_types
    • UploadExportToS3.s3_export_path
    • UploadExportToS3.requires()
    • UploadExportToS3.output()
    • UploadExportToS3.complete()
    • UploadExportToS3.run()
  • DownloadExportFromS3
    • DownloadExportFromS3.local_export_path
    • DownloadExportFromS3.formats
    • DownloadExportFromS3.object_types
    • DownloadExportFromS3.s3_export_path
    • DownloadExportFromS3.parallelism
    • DownloadExportFromS3.requires()
    • DownloadExportFromS3.output()
    • DownloadExportFromS3.complete()
    • DownloadExportFromS3.run()
  • LocalExport
    • LocalExport.local_export_path
    • LocalExport.local_sensitive_export_path
    • LocalExport.formats
    • LocalExport.object_types
    • LocalExport.export_task_type
    • LocalExport.requires()
    • LocalExport.output()
    • LocalExport.complete()
  • AthenaDatabaseTarget
    • AthenaDatabaseTarget.exists()
  • CreateAthena
    • CreateAthena.object_types
    • CreateAthena.s3_export_path
    • CreateAthena.s3_athena_output_location
    • CreateAthena.athena_db_name
    • CreateAthena.requires()
    • CreateAthena.output()
    • CreateAthena.run()
  • RunExportAll
    • RunExportAll.formats
    • RunExportAll.object_types
    • RunExportAll.s3_export_path
    • RunExportAll.s3_athena_output_location
    • RunExportAll.athena_db_name
    • RunExportAll.requires()
• swh.export.relational module
  • BLOOM_FILTER_COLUMNS
• swh.export.utils module
  • ZSTFile
    • ZSTFile.read()
    • ZSTFile.write()
  • SQLiteSet
    • SQLiteSet.add()
  • LevelDBSet
    • LevelDBSet.add()
  • remove_pull_requests()
  • subdirectories_for_object_type()

Module contents#