diff --git a/scripts/us_census/pep/annual_population/golden_data/golden_summary_report.csv b/scripts/us_census/pep/annual_population/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..9a4235850c --- /dev/null +++ b/scripts/us_census/pep/annual_population/golden_data/golden_summary_report.csv @@ -0,0 +1,2 @@ +"MinDate","StatVar","observationPeriods","Units","ScalingFactors","MeasurementMethods","NumPlaces" +"1900","Count_Person","[P1Y]","[]","[]","[CensusPEPSurvey]","23057" diff --git a/scripts/us_census/pep/annual_population/manifest.json b/scripts/us_census/pep/annual_population/manifest.json index 3f1cef3508..7655d3c516 100644 --- a/scripts/us_census/pep/annual_population/manifest.json +++ b/scripts/us_census/pep/annual_population/manifest.json @@ -20,7 +20,8 @@ "cleaned_csv": "output_files/usa_annual_population.csv" } ], - "cron_schedule": "0 10 * * 2" + "cron_schedule": "0 10 * * 2", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/scripts/us_census/pep/annual_population/preprocess.py b/scripts/us_census/pep/annual_population/preprocess.py index 53027a3de2..09841b2062 100644 --- a/scripts/us_census/pep/annual_population/preprocess.py +++ b/scripts/us_census/pep/annual_population/preprocess.py @@ -129,7 +129,7 @@ def _load_data_df(path: str, elif file_format.lower() == "txt": data_df = pd.read_table(path, index_col=False, - delim_whitespace=True, + sep=r'\s+', engine='python', header=header, skiprows=skip_rows) @@ -148,7 +148,7 @@ def _geo_id(val: pd.Series) -> str: str: State/County GeoId """ res = "geoId/" - state, county = val[0], val[1] + state, county = val.iloc[0], val.iloc[1] if county == "000": return res + state return res + state + county diff --git a/scripts/us_census/pep/annual_population/validation_config.json b/scripts/us_census/pep/annual_population/validation_config.json new file mode 100644 index 0000000000..791c40c55e --- /dev/null +++ b/scripts/us_census/pep/annual_population/validation_config.json @@ -0,0 +1,20 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + } + ] +}