diff --git a/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/golden_data/golden_summary_report.csv b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..4c5595abf6 --- /dev/null +++ b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/golden_data/golden_summary_report.csv @@ -0,0 +1,8 @@ +"Units","StatVar","ScalingFactors","observationPeriods","NumPlaces","MeasurementMethods","MinDate" +"[]","Count_Death_AsAFractionOfCount_Person","[]","[]","2122","[EurostatRegionalStatistics]","2000" +"[]","GrowthRate_Count_Person","[]","[]","2122","[EurostatRegionalStatistics]","2000" +"[]","Count_Death","[]","[]","2125","[EurostatRegionalStatistics]","2000" +"[]","Count_Person","[]","[]","2125","[EurostatRegionalStatistics]","2000" +"[]","Count_BirthEvent","[]","[]","2124","[EurostatRegionalStatistics]","2000" +"[]","IncrementalCount_Person","[]","[]","2125","[EurostatRegionalStatistics]","2000" +"[]","Count_BirthEvent_AsAFractionOfCount_Person","[]","[]","2121","[EurostatRegionalStatistics]","2000" diff --git a/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/manifest.json index b5820425f0..2c43154003 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/manifest.json @@ -20,7 +20,8 @@ "demo_r_gind3.tsv.gz", "input_files/input_file.tsv" ], - "cron_schedule": "00 02 * * 2" + "cron_schedule": "00 02 * * 2", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/validation_config.json b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/validation_config.json new file mode 100644 index 0000000000..791c40c55e --- /dev/null +++ b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/validation_config.json @@ -0,0 +1,20 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + } + ] +} diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_attainment/golden_data/golden_summary_report.csv b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..c8b2f0cfa8 --- /dev/null +++ b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/golden_data/golden_summary_report.csv @@ -0,0 +1,7 @@ +"StatVar","MeasurementMethods","ScalingFactors","NumPlaces","MinDate","Units","observationPeriods" +"Count_Person_25To64Years_UpperSecondaryEducationOrHigher_AsAFractionOfCount_Person_25To64Years","[EurostatRegionalStatistics]","[100]","512","2000","[Percent]","[]" +"Count_Person_25To64Years_TertiaryEducation_Female_AsAFractionOfCount_Person_25To64Years_Female","[EurostatRegionalStatistics]","[100]","512","2000","[Percent]","[]" +"Count_Person_25To64Years_TertiaryEducation_Male_AsAFractionOfCount_Person_25To64Years_Male","[EurostatRegionalStatistics]","[100]","512","2000","[Percent]","[]" +"Count_Person_25To64Years_UpperSecondaryEducationOrHigher_Male_AsAFractionOfCount_Person_25To64Years_Male","[EurostatRegionalStatistics]","[100]","512","2000","[Percent]","[]" +"Count_Person_25To64Years_UpperSecondaryEducationOrHigher_Female_AsAFractionOfCount_Person_25To64Years_Female","[EurostatRegionalStatistics]","[100]","512","2000","[Percent]","[]" +"Count_Person_25To64Years_TertiaryEducation_AsAFractionOfCount_Person_25To64Years","[EurostatRegionalStatistics]","[100]","512","2000","[Percent]","[]" diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_attainment/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/manifest.json index a140431d7b..e44fb04003 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/education_attainment/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/manifest.json @@ -20,7 +20,8 @@ "edat_lfse_04.tsv.gz", "input_files/input_file.tsv" ], - "cron_schedule": "30 02 * * 2" + "cron_schedule": "30 02 * * 2", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_attainment/validation_config.json b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/validation_config.json new file mode 100644 index 0000000000..791c40c55e --- /dev/null +++ b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/validation_config.json @@ -0,0 +1,20 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + } + ] +} diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/golden_data/golden_summary_report.csv b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..97a32c014a --- /dev/null +++ b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/golden_data/golden_summary_report.csv @@ -0,0 +1,4 @@ +"Units","MeasurementMethods","ScalingFactors","observationPeriods","NumPlaces","StatVar","MinDate" +"[Percent]","[EurostatRegionalStatistics]","[100]","[]","512","Count_Person_25To64Years_EnrolledInEducationOrTraining_AsAFractionOfCount_Person_25To64Years","2000" +"[Percent]","[EurostatRegionalStatistics]","[100]","[]","509","Count_Person_25To64Years_EnrolledInEducationOrTraining_Male_AsAFractionOfCount_Person_25To64Years_Male","2000" +"[Percent]","[EurostatRegionalStatistics]","[100]","[]","511","Count_Person_25To64Years_EnrolledInEducationOrTraining_Female_AsAFractionOfCount_Person_25To64Years_Female","2000" diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/manifest.json index 60ac5e00a8..8000fbf0c7 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/manifest.json @@ -20,7 +20,8 @@ "trng_lfse_04.tsv.gz", "input_files/input_file.tsv" ], - "cron_schedule": "00 03 * * 2" + "cron_schedule": "00 03 * * 2", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/validation_config.json b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/validation_config.json new file mode 100644 index 0000000000..e3f52fbeeb --- /dev/null +++ b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/validation_config.json @@ -0,0 +1,21 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + } + ] +} +