The articles here below describe the jobs that are used to populate the logs tables according to the Data Ocean - Error Log System.
All the new projects in GCP are required to create these tables in the dataset STG.
CREATE OR REPLACE TABLE STG.run_jobs
(
meta_run_id STRING,
meta_start_date TIMESTAMP,
meta_end_date TIMESTAMP,
meta_status STRING,
meta_execution_id STRING,
meta_error STRING,
job_name STRING
);
CREATE OR REPLACE TABLE STG.reject_files
(
meta_run_id STRING,
meta_md5_hsh STRING ,
meta_content STRING,
meta_error STRING,
meta_domain STRING,
meta_file_name STRING,
meta_file_path STRING,
meta_source_system STRING,
meta_insert_date TIMESTAMP
)
;
CREATE OR REPLACE TABLE STG.log_tables
(
meta_run_id STRING,
meta_source_system STRING,
meta_target_system STRING,
meta_step STRING,
meta_status STRING,
meta_num_lines INT64,
meta_error_lines INT64
);
CREATE OR REPLACE TABLE STG.log_files
(
meta_run_id STRING,
meta_file_name STRING,
meta_bucket_load_date TIMESTAMP,
meta_stg_insert_date TIMESTAMP,
meta_status STRING,
meta_source_system STRING,
meta_periodicity STRING,
meta_extraction_type STRING,
meta_domain STRING,
meta_num_lines INT64,
meta_num_error_lines INT64
)
;