Log jobs

The articles here below describe the jobs that are used to populate the logs tables according to the Data Ocean - Error Log System.

• Write logs_table
• Write runs_jobs log table

All the new projects in GCP are required to create these tables in the dataset STG.

CREATE OR REPLACE TABLE STG.run_jobs
(
  meta_run_id STRING,
  meta_start_date TIMESTAMP,
  meta_end_date TIMESTAMP,
  meta_status STRING,
  meta_execution_id STRING,
  meta_error STRING,
  job_name STRING
);
CREATE OR REPLACE TABLE STG.reject_files
(
  meta_run_id STRING,
  meta_md5_hsh STRING ,
  meta_content STRING,
  meta_error STRING,
  meta_domain STRING,
  meta_file_name STRING,
  meta_file_path STRING,
  meta_source_system STRING,
  meta_insert_date TIMESTAMP
)
;
CREATE OR REPLACE TABLE STG.log_tables
(
  meta_run_id STRING,
  meta_source_system STRING,
  meta_target_system STRING,
  meta_step STRING,
  meta_status STRING,
  meta_num_lines INT64,
  meta_error_lines INT64
);

CREATE OR REPLACE TABLE STG.log_files
(
  meta_run_id STRING,
  meta_file_name STRING,
  meta_bucket_load_date TIMESTAMP,
  meta_stg_insert_date TIMESTAMP,
  meta_status STRING,
  meta_source_system STRING,
  meta_periodicity STRING,
  meta_extraction_type STRING,
  meta_domain STRING,
  meta_num_lines INT64,
  meta_num_error_lines INT64
)
;