diff --git a/Makefile b/Makefile index ff545f4..e68e574 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ CLOUDWATCH_FROM_DATE = $(shell ./scripts/determine-cloudwatch-from-date-based-on CLOUDWATCH_TO_DATE = $(shell date '+%Y-%m-%d') CLOUDWATCH_TARGET_DIR = ./logs/cloudwatch CLOUDWATCH_JSONL_FILE = ./logs/ingress.jsonl +CLOUDWATCH_JSONL_GZ_FILE = $(CLOUDWATCH_JSONL_FILE).gz CLOUDWATCH_JSONL_SCHEMA_FILE = $(CLOUDWATCH_JSONL_FILE).bq-schema.json @@ -84,17 +85,18 @@ download-events-from-s3: "$(CLOUDWATCH_TO_DATE)" \ "$(CLOUDWATCH_TARGET_DIR)" -.convert-cloudwatch-logs-to-jsonl: - ./scripts/convert-cloudwatch-logs-to-jsonl.sh \ +.convert-gzipped-cloudwatch-logs-to-jsonl-gz: + ./scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh \ "$(CLOUDWATCH_TARGET_DIR)" \ - "$(CLOUDWATCH_JSONL_FILE)" + "$(CLOUDWATCH_JSONL_GZ_FILE)" -.generate-schema-for-cloudwatch-jsonl-file: venv - cat "$(CLOUDWATCH_JSONL_FILE)" \ +.generate-schema-for-cloudwatch-jsonl-gz-file: venv + cat "$(CLOUDWATCH_JSONL_GZ_FILE)" \ + | zcat \ | venv/bin/generate-schema \ > "$(CLOUDWATCH_JSONL_SCHEMA_FILE)" -.upload-ingress-jsonl-to-bigquery: +.upload-ingress-jsonl-gz-to-bigquery: bq load \ --project_id=elife-data-pipeline \ --noreplace \ @@ -102,14 +104,14 @@ download-events-from-s3: --schema_update_option=ALLOW_FIELD_ADDITION \ --source_format=NEWLINE_DELIMITED_JSON \ de_proto.sciety_ingress_v1 \ - "$(CLOUDWATCH_JSONL_FILE)" + "$(CLOUDWATCH_JSONL_GZ_FILE)" .do-upload-ingress-logs-from-cloudwatch-to-bigquery: $(MAKE) .cloudwatch-show-info $(MAKE) .export-and-download-from-cloudwatch - $(MAKE) .convert-cloudwatch-logs-to-jsonl - $(MAKE) .generate-schema-for-cloudwatch-jsonl-file - $(MAKE) .upload-ingress-jsonl-to-bigquery + $(MAKE) .convert-gzipped-cloudwatch-logs-to-jsonl-gz + $(MAKE) .generate-schema-for-cloudwatch-jsonl-gz-file + $(MAKE) .upload-ingress-jsonl-gz-to-bigquery .upload-ingress-logs-from-cloudwatch-to-bigquery: @if [ "$(CLOUDWATCH_FROM_DATE)" = "$(CLOUDWATCH_TO_DATE)" ]; then \ diff --git a/scripts/convert-cloudwatch-logs-to-bigquery-jsonl.sh b/scripts/convert-cloudwatch-logs-to-bigquery-jsonl.sh deleted file mode 100755 index 9f52ac2..0000000 --- a/scripts/convert-cloudwatch-logs-to-bigquery-jsonl.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -cat $1 \ - | sed -e 's/[^ ]* //' \ - | jq --compact-output 'del(.kubernetes) | del(.docker)' \ - > $1.jsonl diff --git a/scripts/convert-cloudwatch-logs-to-jsonl.sh b/scripts/convert-cloudwatch-logs-to-jsonl.sh deleted file mode 100755 index daced73..0000000 --- a/scripts/convert-cloudwatch-logs-to-jsonl.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -local_cloudwatch_dir="$1" -target_jsonl_file="$2" - -if [ -z "${local_cloudwatch_dir}" ] || [ -z "${target_jsonl_file}" ]; then - echo "Usage: $0 " - exit 1 -fi - -echo "converting ${local_cloudwatch_dir} to ${target_jsonl_file}" - -find "${local_cloudwatch_dir}" -type 'f' \ - | grep -v jsonl \ - | xargs -n 1 ./scripts/convert-cloudwatch-logs-to-bigquery-jsonl.sh -(find "${local_cloudwatch_dir}" -type 'f' | grep jsonl | xargs cat) > "${target_jsonl_file}" diff --git a/scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh b/scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh new file mode 100755 index 0000000..c62a7d1 --- /dev/null +++ b/scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +log_gz_file="$1" +common_filename=${log_gz_file%.*} +target_jsonl_gz_file="$common_filename.jsonl.gz" + +echo "converting $log_gz_file to $target_jsonl_gz_file" + +cat $log_gz_file \ + | zcat \ + | sed -e 's/[^ ]* //' \ + | jq --compact-output 'del(.kubernetes) | del(.docker)' \ + | gzip - \ + > $target_jsonl_gz_file diff --git a/scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh b/scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh new file mode 100755 index 0000000..515b0d1 --- /dev/null +++ b/scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -euo pipefail + +local_cloudwatch_dir="$1" +target_jsonl_gz_file="$2" + +if [ -z "${local_cloudwatch_dir}" ] || [ -z "${target_jsonl_gz_file}" ]; then + echo "Usage: $0 " + exit 1 +fi + +echo "converting ${local_cloudwatch_dir} to ${target_jsonl_gz_file}" + +find "${local_cloudwatch_dir}" -type 'f' \ + | grep -v jsonl \ + | xargs -n 1 ./scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh + +echo "combining jsonl files to ${target_jsonl_gz_file}" +(find "${local_cloudwatch_dir}" -type 'f' | grep jsonl.gz | xargs gunzip -c | gzip -) > "${target_jsonl_gz_file}" diff --git a/scripts/download-from-cloudwatch.sh b/scripts/download-from-cloudwatch.sh index 87f2f7b..c63bd7a 100755 --- a/scripts/download-from-cloudwatch.sh +++ b/scripts/download-from-cloudwatch.sh @@ -14,4 +14,3 @@ echo "downloading from ${logs_url} to ${target_dir}" mkdir -p "${target_dir}" aws s3 cp --recursive ${logs_url} "${target_dir}" -gunzip -r "${target_dir}"