diff --git a/deployment/docker/Dockerfile b/deployment/docker/Dockerfile new file mode 100644 index 00000000..6700bbc5 --- /dev/null +++ b/deployment/docker/Dockerfile @@ -0,0 +1,36 @@ +FROM amazonlinux:latest + +# Tools +RUN yum install -y shadow-utils curl git openssl11-devel tar gzip gcc make bzip2-devel bzip2-libs ncurses-devel ncurses-lib libffi libffi-devel readline-devel sqlite-devel pyliblzma xz xz-devel xz-libs + +# Worker +RUN adduser --user-group worker +ENV PATH="/home/worker/.local/bin:${PATH}" + +# Copy files +RUN mkdir /app +RUN chown -R worker:worker /app +USER worker +WORKDIR /app +COPY --chown=worker:worker megalista_dataflow megalista_dataflow + +# Python - virtual env +RUN curl https://pyenv.run | bash +RUN /home/worker/.pyenv/bin/pyenv install 3.9.9 +RUN /home/worker/.pyenv/bin/pyenv virtualenv 3.9.9 virtual_env +RUN mv /home/worker/.pyenv/versions/virtual_env /app +ENV VIRTUAL_ENV=/app/virtual_env +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN env +RUN python --version +RUN python -m ensurepip --upgrade +RUN pip install --upgrade pip setuptools wheel + +# Install requirements +RUN pip install --no-warn-script-location -r megalista_dataflow/requirements.txt + +COPY --chown=worker:worker deployment/docker/entrypoint.sh . +COPY --chown=worker:worker deployment/docker/service-account-file.json ./megalista_dataflow/ + +RUN ["chmod", "+x", "entrypoint.sh"] +ENTRYPOINT [ "/app/entrypoint.sh" ] diff --git a/deployment/docker/build.sh b/deployment/docker/build.sh new file mode 100644 index 00000000..4f2ad1ac --- /dev/null +++ b/deployment/docker/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +if [ $# != 1 ]; then + echo "Usage: $0 container_tag" + exit 1 +fi + +echo +echo "${bold}┌──────────────────────────────────┐${reset}" +echo "${bold}│ Megalista Deployment │${reset}" +echo "${bold}└──────────────────────────────────┘${reset}" +echo +echo "${bold}${text_red}This is not an officially supported Google product.${reset}" +echo "${bold}Megalista docker image will be built with the following tag: ${text_green}$1${bold}${reset}" +echo "Update commit info inside code" +sed -i "s/MEGALISTA_VERSION\s*=.*/MEGALISTA_VERSION = '$(git rev-parse HEAD)'/" ../../megalista_dataflow/config/version.py +echo "Build container" +docker build ../../ -t $1 -f Dockerfile +echo "Cleanup" +sed -i "s/MEGALISTA_VERSION\s*=.*/MEGALISTA_VERSION = '\[megalista_version\]'/" ../../megalista_dataflow/config/version.py +echo "${bold}${text_green}Finished. Image build: $1${reset}" diff --git a/deployment/docker/entrypoint.sh b/deployment/docker/entrypoint.sh new file mode 100644 index 00000000..08e6b0b9 --- /dev/null +++ b/deployment/docker/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/bash +echo "Environment variables handling" + +ENV_VARS=`env | grep ^MEGALISTA_.*` +params="" + +for var in $ENV_VARS; +do + IFS='=' + read key value <<< "$var" + key=`echo "${key}" | tr [:upper:] [:lower:]` + if [[ $key == megalista_* ]] + then + params="${params} --${key:10} ${value}" + fi + IFS=' ' +done + +export GOOGLE_APPLICATION_CREDENTIALS=/app/megalista_dataflow/service-account-file.json + +echo "Activating virual environment (python)" +source virtual_env/bin/activate +echo "Running Megalista" +python megalista_dataflow/main.py \ + --runner DirectRunner \ + --direct_num_workers 0 \ + --direct_running_mode multi_threading \ + ${params} \ No newline at end of file diff --git a/deployment/docker/service-account-file.json b/deployment/docker/service-account-file.json new file mode 100644 index 00000000..a7c252b5 --- /dev/null +++ b/deployment/docker/service-account-file.json @@ -0,0 +1,13 @@ +{ + "type": "service_account", + "project_id": "", + "private_key_id": "", + "private_key": "", + "client_email": "", + "client_id": "", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "" + } + \ No newline at end of file diff --git a/megalista_dataflow/config/logging.py b/megalista_dataflow/config/logging.py index ac2bda2c..29040d16 100644 --- a/megalista_dataflow/config/logging.py +++ b/megalista_dataflow/config/logging.py @@ -13,16 +13,18 @@ # limitations under the License. import logging -import sys -from error.logging_handler import LoggingHandler +from optparse import Option +import sys, io, os, traceback +from types import FrameType +from typing import Optional, Tuple, List, Any + +from models.execution import Execution class LoggingConfig: @staticmethod def config_logging(show_lines: bool = False): # If there is a FileHandler, the execution is running on Dataflow # In this scenario, we shouldn't change the formatter - logging_handler = LoggingHandler() - logging.getLogger().addHandler(logging_handler) file_handler = LoggingConfig.get_file_handler() if file_handler is None: log_format = "[%(levelname)s] %(name)s: %(message)s" @@ -36,9 +38,7 @@ def config_logging(show_lines: bool = False): stream_handler = logging.StreamHandler(stream=sys.stderr) logging.getLogger().addHandler(stream_handler) stream_handler.setFormatter(formatter) - - logging_handler.setFormatter(formatter) - + logging.getLogger().setLevel(logging.ERROR) logging.getLogger("megalista").setLevel(logging.INFO) @@ -50,11 +50,6 @@ def get_stream_handler(): def get_file_handler(): return LoggingConfig.get_handler(logging.FileHandler) - - @staticmethod - def get_logging_handler(): - return LoggingConfig.get_handler(LoggingHandler) - @staticmethod def get_handler(type: type): result_handler = None @@ -63,4 +58,119 @@ def get_handler(type: type): result_handler = handler break - return result_handler \ No newline at end of file + return result_handler + +class _LogWrapper: + def __init__(self, name: Optional[str]): + self._name = str(name) + self._logger = logging.getLogger(name) + + def debug(self, msg: str, *args, **kwargs): + self.log(msg, logging.DEBUG, *args, **kwargs) + + def info(self, msg: str, *args, **kwargs): + self.log(msg, logging.INFO, *args, **kwargs) + + def warning(self, msg: str, *args, **kwargs): + self.log(msg, logging.WARNING, *args, **kwargs) + + def error(self, msg: str, *args, **kwargs): + self.log(msg, logging.ERROR, *args, **kwargs) + + def critical(self, msg: str, *args, **kwargs): + self.log(msg, logging.CRITICAL, *args, **kwargs) + + def exception(self, msg: str, *args, **kwargs): + self.log(msg, logging.CRITICAL, *args, **kwargs) + + def log(self, msg: str, level: int, *args, **kwargs): + stacklevel = self._get_stacklevel(**kwargs) + msg = self._get_msg_execution(msg, **kwargs) + msg = self._get_msg_context(msg, **kwargs) + if level >= logging.ERROR: + _add_error(self._name, msg, stacklevel, level, args) + keys_to_remove = ['execution', 'context'] + for key in keys_to_remove: + if key in kwargs: + del kwargs[key] + self._logger.log(level, msg, *args, **self._change_stacklevel(**kwargs)) + + def _change_stacklevel(self, **kwargs): + stacklevel = self._get_stacklevel(**kwargs) + return dict(kwargs, stacklevel = stacklevel) + + def _get_stacklevel(self, **kwargs): + dict_kwargs = dict(kwargs) + stacklevel = 3 + if 'stacklevel' in dict_kwargs: + stacklevel = 2 + dict_kwargs['stacklevel'] + return stacklevel + + def _get_msg_context(self, msg: str, **kwargs): + if 'context' in kwargs: + context = kwargs['context'] + msg = f'[Context: {context}] {msg}' + return msg + + def _get_msg_execution(self, msg: str, **kwargs): + if 'execution' in kwargs: + execution: Execution = kwargs['execution'] + msg = f'[Execution: {execution.source.source_name} -> {execution.destination.destination_name}] {msg}' + return msg + + +def get_logger(name: Optional[str] = None): + return _LogWrapper(name) + +_error_list: List[logging.LogRecord] = [] + +def _add_error(name: str, msg: str, stacklevel: int, level: int, args): + fn, lno, func, sinfo = _get_stack_trace(stacklevel) + _error_list.append(logging.LogRecord(name, level, fn, lno, msg, args, None, func, sinfo)) + +def _get_stack_trace(stacklevel: int, stack_info: bool = True): + # from python logging module + f: Optional[FrameType] = sys._getframe(3) + if f is not None: + f = f.f_back + orig_f = f + while f and stacklevel > 1: + f = f.f_back + stacklevel -= 1 + if not f: + f = orig_f + rv: Tuple[str, int, str, Optional[str]]= ("(unknown file)", 0, "(unknown function)", None) + if f is not None and hasattr(f, "f_code"): + co = f.f_code + sinfo = None + if stack_info: + sio = io.StringIO() + sio.write('Stack (most recent call last):\n') + traceback.print_stack(f, file=sio) + sinfo = sio.getvalue() + if sinfo[-1] == '\n': + sinfo = sinfo[:-1] + sio.close() + rv = (co.co_filename, f.f_lineno, co.co_name, sinfo) + return rv + +def has_errors() -> bool: + return len(_error_list) > 0 + +def error_list() -> List[logging.LogRecord]: + return _error_list + +def get_formatted_error_list() -> Optional[str]: + records = _error_list + if records is not None and len(records) > 0: + message = '' + for i in range(len(records)): + rec = records[i] + message += f'{i+1}. {rec.msg}\n... in {rec.pathname}:{rec.lineno}\n' + return message + else: + return None + +def null_filter(el: Any) -> Any: + get_logger('megalista.LOG').info(f'Logging: {el}') + return el \ No newline at end of file diff --git a/megalista_dataflow/data_sources/base_data_source.py b/megalista_dataflow/data_sources/base_data_source.py index 382fa24e..707019aa 100644 --- a/megalista_dataflow/data_sources/base_data_source.py +++ b/megalista_dataflow/data_sources/base_data_source.py @@ -30,4 +30,12 @@ def retrieve_data(self, executions: ExecutionsGroupedBySource) -> List[DataRowsG raise NotImplementedError("Source Type not implemented. Please check your configuration (sheet / json / firestore).") def write_transactional_info(self, rows, execution): - raise NotImplementedError("Source Type not implemented. Please check your configuration (sheet / json / firestore).") \ No newline at end of file + raise NotImplementedError("Source Type not implemented. Please check your configuration (sheet / json / firestore).") + + @staticmethod + def _convert_row_to_dict(row): + result = {} + for key, value in row.items(): + result[key] = value + return result + \ No newline at end of file diff --git a/megalista_dataflow/data_sources/big_query/big_query_data_source.py b/megalista_dataflow/data_sources/big_query/big_query_data_source.py index 8ba20cd7..8e752344 100644 --- a/megalista_dataflow/data_sources/big_query/big_query_data_source.py +++ b/megalista_dataflow/data_sources/big_query/big_query_data_source.py @@ -11,12 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from sys import exc_info from typing import Any, List, Iterable, Tuple, Dict, Optional from datetime import datetime from string import Template import apache_beam as beam -import logging +from config import logging from google.cloud import bigquery from google.cloud.bigquery import SchemaField, Client from apache_beam.io.gcp.bigquery import ReadFromBigQueryRequest @@ -44,7 +45,7 @@ def __init__(self, if transactional_type is not TransactionalType.NOT_TRANSACTIONAL: if not bq_ops_dataset or bq_ops_dataset == '': - raise Exception(f'Missing bq_ops_dataset for this uploader. Source="{self._source_name}". Destination="{self._destination_name}"') + raise ValueError(f'Missing bq_ops_dataset for this uploader. Source="{self._source_name}". Destination="{self._destination_name}"') def retrieve_data(self, executions: ExecutionsGroupedBySource) -> List[DataRowsGroupedBySource]: @@ -61,11 +62,11 @@ def _retrieve_data_non_transactional(self, executions: ExecutionsGroupedBySource if DataSchemas.validate_data_columns(cols, self._destination_type): cols = DataSchemas.get_cols_names(cols, self._destination_type) query = f"SELECT {','.join(cols)} FROM `{table_name}` AS data" - logging.getLogger(_LOGGER_NAME).info(f'Reading from table {table_name} for Execution {executions}') + logging.get_logger(_LOGGER_NAME).info(f'Reading from table {table_name} for Execution {executions}') elements = [] for row in client.query(query).result(page_size=_BIGQUERY_PAGE_SIZE): - elements.append(_convert_row_to_dict(row)) - logging.getLogger(_LOGGER_NAME).info(f'Data source ({self._source_name}): using {len(elements)} rows') + elements.append(BaseDataSource._convert_row_to_dict(row)) + logging.get_logger(_LOGGER_NAME).info(f'Data source ({self._source_name}): using {len(elements)} rows') return [DataRowsGroupedBySource(executions, elements)] else: @@ -79,7 +80,7 @@ def _retrieve_data_transactional(self, executions: ExecutionsGroupedBySource) -> self._ensure_control_table_exists(client, uploaded_table_name) cols = self._get_table_columns(client, table_name) - logging.getLogger(_LOGGER_NAME).info(f'Destination Type: {self._destination_type}') + logging.get_logger(_LOGGER_NAME).info(f'Destination Type: {self._destination_type}') if DataSchemas.validate_data_columns(cols, self._destination_type): cols = DataSchemas.get_cols_names(cols, self._destination_type) query_cols = ','.join(['data.' + col for col in cols]) @@ -93,15 +94,15 @@ def _retrieve_data_transactional(self, executions: ExecutionsGroupedBySource) -> LEFT JOIN $uploaded_table_name AS uploaded USING(gclid, time) \ WHERE uploaded.gclid IS NULL;" else: - raise Exception(f'Unrecognized TransactionalType: {self._transactional_type}. Source="{self._source_name}". Destination="{self._destination_name}"') + raise ValueError(f'Unrecognized TransactionalType: {self._transactional_type}. Source="{self._source_name}". Destination="{self._destination_name}"') query = Template(template).substitute(table_name=table_name, uploaded_table_name=uploaded_table_name, query_cols=query_cols) - logging.getLogger(_LOGGER_NAME).info( + logging.get_logger(_LOGGER_NAME).info( f'Reading from table `{table_name}` for Execution {executions}') elements = [] for row in client.query(query).result(page_size=_BIGQUERY_PAGE_SIZE): - elements.append(_convert_row_to_dict(row)) - logging.getLogger(_LOGGER_NAME).info(f'Data source ({self._source_name}): using {len(elements)} rows') + elements.append(BaseDataSource._convert_row_to_dict(row)) + logging.get_logger(_LOGGER_NAME).info(f'Data source ({self._source_name}): using {len(elements)} rows') return [DataRowsGroupedBySource(executions, elements)] else: raise ValueError(f'Data source incomplete. {DataSchemas.get_error_message(cols, self._destination_type)} Source="{self._source_name}". Destination="{self._destination_name}"') @@ -122,18 +123,18 @@ def _ensure_control_table_exists(self, client: Client, uploaded_table_name: str) PARTITION BY _PARTITIONDATE \ OPTIONS(partition_expiration_days=15)" else: - raise Exception(f'Unrecognized TransactionalType: {self._transactional_type}. Source="{self._source_name}". Destination="{self._destination_name}"') + raise ValueError(f'Unrecognized TransactionalType: {self._transactional_type}. Source="{self._source_name}". Destination="{self._destination_name}"') query = Template(template).substitute(uploaded_table_name=uploaded_table_name) - logging.getLogger(_LOGGER_NAME).info( + logging.get_logger(_LOGGER_NAME).info( f"Creating table `{uploaded_table_name}` if it doesn't exist") client.query(query).result() def write_transactional_info(self, rows, execution: Execution): if len(rows) == 0: - logging.getLogger(_LOGGER_NAME).info( + logging.get_logger(_LOGGER_NAME).info( "No rows to insert. Skipping...") else: table_name = self._get_table_name(execution.source.source_metadata, True) @@ -152,7 +153,7 @@ def write_transactional_info(self, rows, execution: Execution): for results in partial_results: for result in results: - logging.getLogger(_LOGGER_NAME).error(result['errors']) + logging.get_logger(_LOGGER_NAME).error(result['errors'], execution=execution) def _get_now(self): return datetime.now().timestamp() @@ -173,14 +174,14 @@ def _get_schema_fields(self): return SchemaField("uuid", "string"), SchemaField("timestamp", "timestamp") if self._transactional_type == TransactionalType.GCLID_TIME: return SchemaField("gclid", "string"), SchemaField("time", "string"), SchemaField("timestamp", "timestamp") - raise Exception(f'Unrecognized TransactionalType: {self._transactional_type}. Source="{self._source_name}". Destination="{self._destination_name}"') + raise ValueError(f'Unrecognized TransactionalType: {self._transactional_type}. Source="{self._source_name}". Destination="{self._destination_name}"') def _get_bq_rows(self, rows, now): if self._transactional_type == TransactionalType.UUID: return [{'uuid': row['uuid'], 'timestamp': now} for row in rows] if self._transactional_type == TransactionalType.GCLID_TIME: return [{'gclid': row['gclid'], 'time': row['time'], 'timestamp': now} for row in rows] - raise Exception(f'Unrecognized TransactionalType: {self._transactional_type}. Source="{self._source_name}". Destination="{self._destination_name}"') + raise ValueError(f'Unrecognized TransactionalType: {self._transactional_type}. Source="{self._source_name}". Destination="{self._destination_name}"') def _get_table_columns(self, client, table_name): table = client.get_table(table_name) @@ -188,10 +189,4 @@ def _get_table_columns(self, client, table_name): def _get_bq_client(self): return bigquery.Client(location=self._bq_location) - -def _convert_row_to_dict(row): - dict = {} - for key, value in row.items(): - dict[key] = value - return dict - \ No newline at end of file + \ No newline at end of file diff --git a/megalista_dataflow/data_sources/big_query/big_query_data_source_test.py b/megalista_dataflow/data_sources/big_query/big_query_data_source_test.py index 4cede48f..88f4c012 100644 --- a/megalista_dataflow/data_sources/big_query/big_query_data_source_test.py +++ b/megalista_dataflow/data_sources/big_query/big_query_data_source_test.py @@ -22,6 +22,7 @@ from models.execution import SourceType from models.execution import Batch import pytest +import logging from models.execution import TransactionalType diff --git a/megalista_dataflow/data_sources/data_schemas.py b/megalista_dataflow/data_sources/data_schemas.py index 2b1adbce..cf6e1b78 100644 --- a/megalista_dataflow/data_sources/data_schemas.py +++ b/megalista_dataflow/data_sources/data_schemas.py @@ -13,15 +13,19 @@ # limitations under the License. -from configparser import MissingSectionHeaderError from typing import List, Dict, Any from models.execution import Destination, DestinationType, Execution, Batch -import logging +from config import logging import functools import pandas as pd import ast import re +CUSTOM_VARIABLES_TYPE = 'customVariables.type' +CUSTOM_VARIABLES_VALUE = 'customVariables.value' +CUSTOM_DIMENSION = 'cd\\d+' +CUSTOM_METRIC = 'cm\\d+' + _dtypes: Dict[str, Dict[str, Any]] = { 'CM_OFFLINE_CONVERSION': { 'columns' : [ @@ -33,8 +37,8 @@ {'name': 'value', 'required': False, 'data_type': 'int'}, {'name': 'quantity', 'required': False, 'data_type': 'int'}, {'name': 'timestamp', 'required': False, 'data_type': 'string'}, - {'name': 'customVariables.type', 'required': False, 'data_type': 'string'}, - {'name': 'customVariables.value', 'required': False, 'data_type': 'string'} + {'name': CUSTOM_VARIABLES_TYPE, 'required': False, 'data_type': 'string'}, + {'name': CUSTOM_VARIABLES_VALUE, 'required': False, 'data_type': 'string'} ], 'groups': [ ['gclid', 'mobileDeviceId', 'encryptedUserId', 'matchId'] @@ -84,8 +88,8 @@ {'name': 'phone', 'required': False, 'data_type': 'string'}, {'name': 'mailing_address_first_name', 'required': False, 'data_type': 'string'}, {'name': 'mailing_address_last_name', 'required': False, 'data_type': 'string'}, - {'name': 'mailing_address_country_name', 'required': False, 'data_type': 'string'}, - {'name': 'mailing_address_zip_name', 'required': False, 'data_type': 'string'} + {'name': 'mailing_address_country', 'required': False, 'data_type': 'string'}, + {'name': 'mailing_address_zip', 'required': False, 'data_type': 'string'} ], 'groups': [] }, @@ -134,8 +138,8 @@ {'name': 'event_action', 'required': True, 'data_type': 'string'}, {'name': 'event_label', 'required': False, 'data_type': 'string'}, {'name': 'event_value', 'required': False, 'data_type': 'string'}, - {'name': 'cm\\d+', 'required': False, 'data_type': 'string'}, - {'name': 'cd\\d+', 'required': False, 'data_type': 'string'}, + {'name': CUSTOM_METRIC, 'required': False, 'data_type': 'string'}, + {'name': CUSTOM_DIMENSION, 'required': False, 'data_type': 'string'}, {'name': 'campaign_source', 'required': False, 'data_type': 'string'}, {'name': 'campaign_medium', 'required': False, 'data_type': 'string'}, ], @@ -145,9 +149,9 @@ }, 'GA_DATA_IMPORT': { 'columns': [ - {'name': 'cd\\d+', 'required': True, 'data_type': 'string'}, - {'name': 'cd\\d+', 'required': True, 'data_type': 'string'}, - {'name': 'cd\\d+', 'required': False, 'data_type': 'string'}, + {'name': CUSTOM_DIMENSION, 'required': True, 'data_type': 'string'}, + {'name': CUSTOM_DIMENSION, 'required': True, 'data_type': 'string'}, + {'name': CUSTOM_DIMENSION, 'required': False, 'data_type': 'string'}, ], 'groups': [] }, @@ -170,8 +174,8 @@ {'name': 'phone', 'required': False, 'data_type': 'string'}, {'name': 'mailing_address_first_name', 'required': False, 'data_type': 'string'}, {'name': 'mailing_address_last_name', 'required': False, 'data_type': 'string'}, - {'name': 'mailing_address_country_name', 'required': False, 'data_type': 'string'}, - {'name': 'mailing_address_zip_name', 'required': False, 'data_type': 'string'} + {'name': 'mailing_address_country', 'required': False, 'data_type': 'string'}, + {'name': 'mailing_address_zip', 'required': False, 'data_type': 'string'} ], 'groups': [] }, @@ -267,23 +271,23 @@ def get_cols_names(data_cols: list, destination_type: DestinationType) -> list: filtered_cols = [] for col in data_cols: - found = False for data_type_col in data_type_cols: if re.match(f'^{data_type_col}$', col) is not None: if col not in filtered_cols: filtered_cols.append(col) - + return filtered_cols # Parse columns that aren't string def update_data_types_not_string(df: pd.DataFrame, destination_type: DestinationType) -> pd.DataFrame: - # temp_dtypes_to_change = _dtypes_not_string[destination_type.name] data_type = _dtypes[destination_type.name] cols_to_change = [col['name'] for col in filter(lambda col: col['data_type'] != 'string', data_type['columns'])] dtypes_to_change = {} for key in cols_to_change: if key in df.columns: - dtypes_to_change[key] = list(filter(lambda col: col['name'] == key, data_type['columns']))[0]['data_type'] + for col in data_type['columns']: + if col['name'] == key: + dtypes_to_change[key] = col['data_type'] return df.astype(dtypes_to_change) @@ -296,8 +300,8 @@ def process_by_destination_type(df: pd.DataFrame, destination_type: DestinationT # Data treatment - CM_OFFLINE_CONVERSION def _join_custom_variables(df) -> pd.DataFrame: - df['customVariables'] = '{ "type": "' + df['customVariables.type'] + '", "value": "' + df['customVariables.value'] + '"}' - df.drop(['customVariables.type', 'customVariables.value'], axis=1, inplace=True) + df['customVariables'] = '{ "type": "' + df[CUSTOM_VARIABLES_TYPE] + '", "value": "' + df[CUSTOM_VARIABLES_VALUE] + '"}' + df.drop([CUSTOM_VARIABLES_TYPE, CUSTOM_VARIABLES_VALUE], axis=1, inplace=True) df['customVariables'] = df.groupby('uuid')['customVariables'].transform(lambda x: '[' + ', '.join(x) + ']') df = df.drop_duplicates() df = df.reset_index() diff --git a/megalista_dataflow/data_sources/data_source.py b/megalista_dataflow/data_sources/data_source.py index e3f315f5..9895a979 100644 --- a/megalista_dataflow/data_sources/data_source.py +++ b/megalista_dataflow/data_sources/data_source.py @@ -27,7 +27,6 @@ class DataSource: @staticmethod def get_data_source(executions: ExecutionsGroupedBySource, transactional_type: TransactionalType, dataflow_options: DataflowOptions) -> BaseDataSource: - data_source = None source_type = executions.source.source_type if source_type == SourceType.BIG_QUERY: bq_ops_dataset = '' @@ -40,5 +39,4 @@ def get_data_source(executions: ExecutionsGroupedBySource, transactional_type: T elif source_type == SourceType.FILE: return FileDataSource(executions, transactional_type, dataflow_options) else: - raise NotImplementedError("Source Type not implemented. Please check your configuration (sheet / json / firestore).") - return data_source + raise NotImplementedError("Source Type not implemented. Please check your configuration (sheet / json / firestore).") \ No newline at end of file diff --git a/megalista_dataflow/data_sources/file/file_data_source.py b/megalista_dataflow/data_sources/file/file_data_source.py index cdc465d7..477d1ca4 100644 --- a/megalista_dataflow/data_sources/file/file_data_source.py +++ b/megalista_dataflow/data_sources/file/file_data_source.py @@ -25,7 +25,7 @@ from apache_beam.typehints.decorators import with_output_types import numpy as np -import logging +from config import logging from models.execution import SourceType, DestinationType, Execution, Batch, TransactionalType, ExecutionsGroupedBySource, DataRowsGroupedBySource from models.options import DataflowOptions @@ -54,10 +54,10 @@ def retrieve_data(self, executions: ExecutionsGroupedBySource) -> List[Any]: def _retrieve_data_non_transactional(self, executions: ExecutionsGroupedBySource) -> List[DataRowsGroupedBySource]: source = executions.source # Get Data Source - data_source = self._get_data_source(source.source_name, source.source_metadata[0]) + data_source = self._get_data_source(source.source_metadata[0]) # Get Data Frame - df = data_source.get_data_frame(source.source_name, source.source_metadata[1]) - logging.getLogger(_LOGGER_NAME).info(f'Data source ({self._source_name}): using {len(df.index)} rows') + df = data_source.get_data_frame(source.source_metadata[1]) + logging.get_logger(_LOGGER_NAME).info(f'Data source ({self._source_name}): using {len(df.index)} rows') if df is not None: df = df.fillna(np.nan).replace([np.nan], [None]) # Process Data Frame @@ -66,16 +66,16 @@ def _retrieve_data_non_transactional(self, executions: ExecutionsGroupedBySource elements.append(FileDataSource._convert_row_to_dict(row)) return [DataRowsGroupedBySource(executions, elements)] else: - raise Exception(f'Unable to read from data source. Source="{source.source_name}".') + raise OSError(f'Unable to read from data source. Source="{source.source_name}".') def _retrieve_data_transactional(self, executions: ExecutionsGroupedBySource) -> List[DataRowsGroupedBySource]: source = executions.source # Get Data Source - data_source = self._get_data_source(source.source_name, source.source_metadata[0]) + data_source = self._get_data_source(source.source_metadata[0]) # Get Data Frame - df = data_source.get_data_frame(source.source_name, source.source_metadata[1]) + df = data_source.get_data_frame(source.source_metadata[1]) # Get Uploaded Data Frame - df_uploaded = data_source.get_data_frame(source.source_name, source.source_metadata[1], is_uploaded=True) + df_uploaded = data_source.get_data_frame(source.source_metadata[1], is_uploaded=True) if df is not None: # Get items that haven't been processed yet @@ -83,19 +83,19 @@ def _retrieve_data_transactional(self, executions: ExecutionsGroupedBySource) -> df_distinct = df_merged.drop(df_merged[df_merged.timestamp.notnull()].index) # Process Data Frame df_distinct = df_distinct.fillna(np.nan).replace([np.nan], [None]) - logging.getLogger(_LOGGER_NAME).info(f'Data source ({self._source_name}): using {len(df_distinct.index)} rows') + logging.get_logger(_LOGGER_NAME).info(f'Data source ({self._source_name}): using {len(df_distinct.index)} rows') elements = [] for index, row in df_distinct.iterrows(): elements.append(FileDataSource._convert_row_to_dict(row)) return [DataRowsGroupedBySource(executions, elements)] else: - raise Exception(f'Unable to read from data source. Source="{self._source_name}". Destination="{self._destination_name}"') + raise OSError(f'Unable to read from data source. Source="{self._source_name}". Destination="{self._destination_name}"') def write_transactional_info(self, rows, execution: Execution): # Get Data Source - data_source = self._get_data_source(execution.source.source_name, execution.source.source_metadata[0]) + data_source = self._get_data_source(execution.source.source_metadata[0]) # Get Data Frame - df = data_source.get_data_frame(execution.source.source_name, execution.source.source_metadata[1], is_uploaded=True) + df = data_source.get_data_frame(execution.source.source_metadata[1], is_uploaded=True) now = datetime.now() @@ -109,10 +109,10 @@ def write_transactional_info(self, rows, execution: Execution): # Upload file # Add _uploaded into path path = FileDataSource._append_filename_uploaded(execution.source.source_metadata[1]) - bytes = data_source._get_file_from_data_frame(df).getbuffer().tobytes() - FileProvider(path, self._dataflow_options, self._source_type, self._source_name, False).write(bytes) + file_bytes = data_source._get_file_from_data_frame(df).getbuffer().tobytes() + FileProvider(path, self._dataflow_options, self._source_type, self._source_name, False).write(file_bytes) - def get_data_frame(self, source_name: str, path: str, is_uploaded: bool = False) -> pd.DataFrame: + def get_data_frame(self, path: str, is_uploaded: bool = False) -> pd.DataFrame: # Change filename if uploaded if is_uploaded: # Add _uploaded into path @@ -149,19 +149,13 @@ def _append_filename_uploaded(path: str) -> str: base_path = base_path + '_uploaded' return base_path + os.path.splitext(path)[1] - def _get_data_source(self, source_name: str, file_type: str): + def _get_data_source(self, file_type: str): file_type = file_type.upper() if file_type == 'PARQUET': return ParquetDataSource(self._executions, self._transactional_type, self._dataflow_options) elif file_type == 'CSV': return CSVDataSource(self._executions, self._transactional_type, self._dataflow_options) raise ValueError(f'Data source not found. Please check your source in config (value={file_type}). Source="{self._source_name}". Destination="{self._destination_name}"') - - def _convert_row_to_dict(row): - dict = {} - for key, value in row.items(): - dict[key] = value - return dict def _update_dtypes(self, destination_type: DestinationType, col_names: list) -> dict: types_dict = DataSchemas.get_data_type(destination_type.name) diff --git a/megalista_dataflow/data_sources/file/file_provider.py b/megalista_dataflow/data_sources/file/file_provider.py index 2934bb35..daaa5b8e 100644 --- a/megalista_dataflow/data_sources/file/file_provider.py +++ b/megalista_dataflow/data_sources/file/file_provider.py @@ -20,7 +20,7 @@ """ import io -import logging +from config import logging from os.path import exists from urllib.parse import ParseResultBytes @@ -52,7 +52,6 @@ def write(self, data): return self._provider.write(data) def _define_file_provider(self): - file_provider = None if self._path.startswith('s3://'): #S3 return self._S3FileProvider(self._path, self._dataflow_options, self._can_skip_read, self._source_name) @@ -60,7 +59,7 @@ def _define_file_provider(self): #GCP Storage #- https is for keeping consistency with previous implementation of JSON Config. return self._GCSFileProvider(self._path, self._dataflow_options, self._can_skip_read, self._source_name) - elif self._path.startswith('file://') or not '://' in self._path: + elif self._path.startswith('file://') or '://' not in self._path: #Local File return self._LocalFileProvider(self._path, self._dataflow_options, self._can_skip_read, self._source_name) raise NotImplementedError(f'Could not define File Provider. Path="{self._path}". Source="{self._source_name}"') @@ -128,7 +127,7 @@ def __init__(self, path: str, dataflow_options: DataflowOptions, can_skip_read: self._s3_client = boto3.client('s3') self._s3_resource = boto3.resource('s3') - logging.getLogger(_LOGGER_NAME).info(f'S3 File Provider initiated. Bucket: "{bucket_name}". Key="{key}"') + logging.get_logger(_LOGGER_NAME).info(f'S3 File Provider initiated. Bucket: "{bucket_name}". Key="{key}"') def read(self): try: @@ -149,7 +148,7 @@ def read(self): return response['Body'].read() def write(self, data): - response = self._s3_client.put_object( + self._s3_client.put_object( Bucket=self._bucket_name, Key=self._key, Body=data @@ -166,7 +165,7 @@ def __init__(self, path: str, dataflow_options: DataflowOptions, can_skip_read: file_path = '/'.join(path.split('/')[1:]) self._bucket_name = bucket_name self._file_path = file_path - logging.getLogger(_LOGGER_NAME).info(f'GCP Storage File Provider initiated. Bucket: "{bucket_name}". Path="{file_path}"') + logging.get_logger(_LOGGER_NAME).info(f'GCP Storage File Provider initiated. Bucket: "{bucket_name}". Path="{file_path}"') self._gcs_client = storage.Client() diff --git a/megalista_dataflow/error/error_handling.py b/megalista_dataflow/error/error_handling.py index 8e962c19..ae883f60 100644 --- a/megalista_dataflow/error/error_handling.py +++ b/megalista_dataflow/error/error_handling.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import base64 -import logging +from config import logging from email.mime.text import MIMEText from typing import Iterable, Optional, Dict @@ -91,7 +91,7 @@ def _should_notify(self): return should_notify.lower() == 'true' def notify(self, destination_type: DestinationType, errors: Iterable[Error]): - logger = logging.getLogger('megalista.GmailNotifier') + logger = logging.get_logger('megalista.GmailNotifier') if not self._should_notify(): logger.info(f'Skipping sending emails notifying of errors: {", ".join(map(str, errors))}') return @@ -163,7 +163,7 @@ def add_error(self, execution: Execution, error_message: str): if execution.destination.destination_type != self._destination_type: raise ValueError( - f'Received a error of destination type: {execution.destination.destination_type}' + f'Received an error of destination type: {execution.destination.destination_type}' f' but this error handler is initialized with {self._destination_type} destination type') error = Error(execution, error_message) diff --git a/megalista_dataflow/error/error_handling_test.py b/megalista_dataflow/error/error_handling_test.py index c8a78fce..95aaada3 100644 --- a/megalista_dataflow/error/error_handling_test.py +++ b/megalista_dataflow/error/error_handling_test.py @@ -20,6 +20,14 @@ from models.execution import DestinationType, Execution, AccountConfig, Source, SourceType, Destination from models.oauth_credentials import OAuthCredentials +FIRST_EMAIL = 'a@a.com' +SECOND_EMAIL = 'b@b.com' +THIRD_EMAIL = 'c@c.com' +ERROR_MESSAGE = 'error message' +ERROR_FIRST_EXECUTION_FIRST_INPUT = 'Error for first execution, first input' +ERROR_FIRST_EXECUTION_SECOND_INPUT = 'Error for first execution, second input' +ERROR_SECOND_EXECUTION_FIRST_INPUT = 'Error for second execution, first input' + class MockErrorNotifier(ErrorNotifier): def __init__(self): @@ -48,9 +56,9 @@ def test_single_error_per_execution(): first_execution = create_execution('source1', 'destination1') second_execution = create_execution('source1', 'destination2') - error_handler.add_error(first_execution, 'Error for first execution, fist input') - error_handler.add_error(first_execution, 'Error for first execution, second input') - error_handler.add_error(second_execution, 'Error for second execution, fist input') + error_handler.add_error(first_execution, ERROR_FIRST_EXECUTION_FIRST_INPUT) + error_handler.add_error(first_execution, ERROR_FIRST_EXECUTION_SECOND_INPUT) + error_handler.add_error(second_execution, ERROR_SECOND_EXECUTION_FIRST_INPUT) returned_errors = error_handler.errors assert len(returned_errors) == 2 @@ -62,7 +70,7 @@ def test_destination_type_consistency(): wrong_destination_type_execution = create_execution('source', 'destination') with pytest.raises(ValueError): - error_handler.add_error(wrong_destination_type_execution, 'error message') + error_handler.add_error(wrong_destination_type_execution, ERROR_MESSAGE) def test_errors_sent_to_error_notifier(): @@ -72,14 +80,14 @@ def test_errors_sent_to_error_notifier(): first_execution = create_execution('source1', 'destination1') second_execution = create_execution('source1', 'destination2') - error_handler.add_error(first_execution, 'Error for first execution, fist input') - error_handler.add_error(second_execution, 'Error for second execution, fist input') + error_handler.add_error(first_execution, ERROR_FIRST_EXECUTION_FIRST_INPUT) + error_handler.add_error(second_execution, ERROR_SECOND_EXECUTION_FIRST_INPUT) error_handler.notify_errors() assert mock_notifier.were_errors_sent is True - assert mock_notifier.errors_sent == {first_execution: 'Error for first execution, fist input', - second_execution: 'Error for second execution, fist input'} + assert mock_notifier.errors_sent == {first_execution: ERROR_FIRST_EXECUTION_FIRST_INPUT, + second_execution: ERROR_SECOND_EXECUTION_FIRST_INPUT} assert mock_notifier.destination_type == DestinationType.ADS_OFFLINE_CONVERSION @@ -95,38 +103,26 @@ def test_should_not_notify_when_empty_errors(): # GmailNotifier tests def test_multiple_destinations_separated_by_comma(): - first_email = 'a@a.com' - second_email = 'b@b.com' - third_email = 'c@c.com' - credentials = OAuthCredentials('', '', '', '') gmail_notifier = GmailNotifier(StaticValueProvider(str, 'true'), credentials, - StaticValueProvider(str, f'{first_email}, {second_email} ,{third_email}')) + StaticValueProvider(str, f'{FIRST_EMAIL}, {SECOND_EMAIL} ,{THIRD_EMAIL}')) emails = set(gmail_notifier.email_destinations) assert len(emails) == 3 - assert set(emails) == {first_email, third_email, second_email} + assert set(emails) == {FIRST_EMAIL, THIRD_EMAIL, SECOND_EMAIL} def test_should_not_notify_when_param_is_false(): - first_email = 'a@a.com' - second_email = 'b@b.com' - third_email = 'c@c.com' - credentials = OAuthCredentials('', '', '', '') gmail_notifier = GmailNotifier(StaticValueProvider(str, 'false'), credentials, - StaticValueProvider(str, f'{first_email}, {second_email} ,{third_email}')) + StaticValueProvider(str, f'{FIRST_EMAIL}, {SECOND_EMAIL} ,{THIRD_EMAIL}')) - gmail_notifier.notify(DestinationType.ADS_OFFLINE_CONVERSION, [Error(create_execution('s', 'd'), 'error message')]) + gmail_notifier.notify(DestinationType.ADS_OFFLINE_CONVERSION, [Error(create_execution('s', 'd'), ERROR_MESSAGE)]) def test_should_not_notify_when_param_is_empty(): - first_email = 'a@a.com' - second_email = 'b@b.com' - third_email = 'c@c.com' - credentials = OAuthCredentials('', '', '', '') gmail_notifier = GmailNotifier(StaticValueProvider(str, None), credentials, - StaticValueProvider(str, f'{first_email}, {second_email} ,{third_email}')) + StaticValueProvider(str, f'{FIRST_EMAIL}, {SECOND_EMAIL} ,{THIRD_EMAIL}')) - gmail_notifier.notify(DestinationType.ADS_OFFLINE_CONVERSION, [Error(create_execution('s', 'd'), 'error message')]) + gmail_notifier.notify(DestinationType.ADS_OFFLINE_CONVERSION, [Error(create_execution('s', 'd'), ERROR_MESSAGE)]) diff --git a/megalista_dataflow/error/logging_handler.py b/megalista_dataflow/error/logging_handler.py deleted file mode 100644 index 79ab20c1..00000000 --- a/megalista_dataflow/error/logging_handler.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import re -from typing import Optional, List -from .error_handling import ErrorNotifier - -class LoggingHandler(logging.Handler): - def __init__(self, level=logging.INFO): - self.level = level - self.filters = [] - self.lock = None - self._has_errors:bool = False - self._records: List[logging.LogRecord] = [] - - def emit(self, record: logging.LogRecord): - if record.levelno >= logging.ERROR: - self._has_errors = True - self._records.append(record) - - @property - def has_errors(self) -> bool: - return self._has_errors - - @property - def all_records(self) -> List[logging.LogRecord]: - return self._records - - @property - def error_records(self) -> List[logging.LogRecord]: - return list(filter(lambda rec: rec.levelno >= logging.ERROR, self._records)) - - @staticmethod - def format_records(records: List[logging.LogRecord]) -> Optional[str]: - if records is not None and len(records) > 0: - message = '' - for i in range(len(records)): - rec = records[i] - message += f'{i+1}. {rec.msg}\n in {rec.pathname}:{rec.lineno}\n' - return message - else: - return None \ No newline at end of file diff --git a/megalista_dataflow/error/logging_handler_test.py b/megalista_dataflow/error/logging_handler_test.py deleted file mode 100644 index 41d38ea8..00000000 --- a/megalista_dataflow/error/logging_handler_test.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import Iterable -from error.logging_handler import LoggingHandler -import logging - -import pytest -from apache_beam.options.value_provider import StaticValueProvider - -from error.error_handling import ErrorHandler, Error, GmailNotifier, ErrorNotifier -from models.execution import DestinationType, Execution, AccountConfig, Source, SourceType, Destination -from models.oauth_credentials import OAuthCredentials - -def get_log_record_info(): - return logging.LogRecord("unit_test", logging.INFO, '', 1, 'Message Info', None, None) - -def get_log_record_error(): - return logging.LogRecord("unit_test", logging.ERROR, '', 1, 'Message Error', None, None) - -# ErrorHandler tests -def test_has_errors(): - handler = LoggingHandler() - assert handler.has_errors == False - handler.emit(get_log_record_info()) - assert handler.has_errors == False - handler.emit(get_log_record_error()) - assert handler.has_errors == True - diff --git a/megalista_dataflow/main.py b/megalista_dataflow/main.py index d658a48f..a57f6169 100644 --- a/megalista_dataflow/main.py +++ b/megalista_dataflow/main.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging import warnings import apache_beam as beam @@ -41,7 +41,7 @@ "ignore", "Your application has authenticated using end user credentials" ) -def run(argv=None): +def run(): pipeline_options = PipelineOptions() dataflow_options = pipeline_options.view_as(DataflowOptions) oauth_credentials = OAuthCredentials( @@ -77,20 +77,14 @@ def run(argv=None): processing_results = executions | "Execute integrations" >> ProcessingStep(params) - # todo: update trix at the end - tuple(processing_results) | "Consolidate results" >> LastStep(params) if __name__ == "__main__": run() - logging_handler = LoggingConfig.get_logging_handler() - if logging_handler is None: - logging.getLogger("megalista").info(f"MEGALISTA build {MEGALISTA_VERSION}: Clould not find error interception handler. Skipping error intereception.") + if logging.has_errors(): + logging.get_logger("megalista").critical(f'MEGALISTA build {MEGALISTA_VERSION}: Completed with errors') + raise SystemExit(1) else: - if logging_handler.has_errors: - logging.getLogger("megalista").critical(f'MEGALISTA build {MEGALISTA_VERSION}: Completed with errors') - raise SystemExit(1) - else: - logging.getLogger("megalista").info(f"MEGALISTA build {MEGALISTA_VERSION}: Completed successfully!") + logging.get_logger("megalista").info(f"MEGALISTA build {MEGALISTA_VERSION}: Completed successfully!") exit(0) diff --git a/megalista_dataflow/mappers/abstract_list_pii_hashing_mapper.py b/megalista_dataflow/mappers/abstract_list_pii_hashing_mapper.py index 9701ae61..d2b28519 100644 --- a/megalista_dataflow/mappers/abstract_list_pii_hashing_mapper.py +++ b/megalista_dataflow/mappers/abstract_list_pii_hashing_mapper.py @@ -13,7 +13,7 @@ # limitations under the License. import hashlib -import logging +from config import logging import re from models.execution import Batch @@ -26,14 +26,15 @@ def __init__(self, should_hash_fields): def hash_field(self, field): if self.should_hash_fields: - return hashlib.sha256(field.strip().lower().encode("utf-8")).hexdigest() + msg = field.strip().lower().encode("utf-8") + return hashlib.sha256(msg).hexdigest() return field class ListPIIHashingMapper: def __init__(self): - self.logger = logging.getLogger( + self.logger = logging.get_logger( "megalista.AbstractListPIIHashingMapper") def _get_default_hasheable_keys(self): diff --git a/megalista_dataflow/mappers/ads_user_list_pii_hashing_mapper.py b/megalista_dataflow/mappers/ads_user_list_pii_hashing_mapper.py index 0e7375ac..27f87658 100644 --- a/megalista_dataflow/mappers/ads_user_list_pii_hashing_mapper.py +++ b/megalista_dataflow/mappers/ads_user_list_pii_hashing_mapper.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging from models.execution import Batch from mappers.abstract_list_pii_hashing_mapper import ListPIIHashingMapper @@ -20,7 +20,7 @@ class AdsUserListPIIHashingMapper(ListPIIHashingMapper): def __init__(self): - self.logger = logging.getLogger( + self.logger = logging.get_logger( "megalista.AdsUserListPIIHashingMapper") def _hash_user(self, user, hasher): @@ -36,7 +36,7 @@ def _hash_user(self, user, hasher): processed_email = self.normalize_email(user["email"]) processed_user["hashed_email"] = hasher.hash_field( processed_email) - except: + except Exception: self.logger.error(f"Error hashing email for user: {str(user)}") try: @@ -56,14 +56,14 @@ def _hash_user(self, user, hasher): "country_code": user["mailing_address_country"], "postal_code": user["mailing_address_zip"], } - except: + except Exception: self.logger.error(f"Error hashing address for user: {str(user)}") try: if self._is_data_present(user, "phone"): processed_user["hashed_phone_number"] = hasher.hash_field( user["phone"]) - except: + except Exception: self.logger.error(f"Error hashing phone for user: {str(user)}") if self._is_data_present(user, "mobile_device_id"): @@ -73,7 +73,7 @@ def _hash_user(self, user, hasher): if self._is_data_present(user, "user_id"): processed_user["third_party_user_id"] = hasher.hash_field( user["user_id"]) - except: + except Exception: self.logger.error(f"Error hashing user_id for user: {str(user)}") return processed_user diff --git a/megalista_dataflow/mappers/ads_user_list_pii_hashing_mapper_test.py b/megalista_dataflow/mappers/ads_user_list_pii_hashing_mapper_test.py index a7c5d5a5..9d80fdf7 100644 --- a/megalista_dataflow/mappers/ads_user_list_pii_hashing_mapper_test.py +++ b/megalista_dataflow/mappers/ads_user_list_pii_hashing_mapper_test.py @@ -15,6 +15,7 @@ from mappers.ads_user_list_pii_hashing_mapper import AdsUserListPIIHashingMapper from models.execution import Batch +PHONE_1 = "+551199999999" def test_get_should_hash_fields(): @@ -40,7 +41,7 @@ def test_pii_hashing(mocker): users = [{ "email": "john@doe.com", - "phone": "+551199999999", + "phone": PHONE_1, "mailing_address_first_name": "John ", "mailing_address_last_name": "Doe", "mailing_address_zip": "12345", @@ -84,11 +85,11 @@ def test_pii_hashing(mocker): }, { "email": "ca.us@gmail.com", - "phone": "+551199999999", + "phone": PHONE_1, }, { "email": "us.ca@doe.com", - "phone": "+551199999999", + "phone": PHONE_1, } ] diff --git a/megalista_dataflow/mappers/batches_grouped_by_source_mapper.py b/megalista_dataflow/mappers/batches_grouped_by_source_mapper.py index 05551df1..eb0431a1 100644 --- a/megalista_dataflow/mappers/batches_grouped_by_source_mapper.py +++ b/megalista_dataflow/mappers/batches_grouped_by_source_mapper.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging from models.execution import Batch from mappers.abstract_list_pii_hashing_mapper import ListPIIHashingMapper @@ -22,7 +22,7 @@ class BatchesGroupedBySourceMapper(): def __init__(self): - self.logger = logging.getLogger("megalista.BatchesGroupedBySourceMapper") + self.logger = logging.get_logger("megalista.BatchesGroupedBySourceMapper") def encapsulate(self, element): return BatchesGroupedBySource(element[0], element[1]) diff --git a/megalista_dataflow/mappers/dv_user_list_pii_hashing_mapper.py b/megalista_dataflow/mappers/dv_user_list_pii_hashing_mapper.py index 0de6266e..31da58ea 100644 --- a/megalista_dataflow/mappers/dv_user_list_pii_hashing_mapper.py +++ b/megalista_dataflow/mappers/dv_user_list_pii_hashing_mapper.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging from models.execution import Batch from mappers.abstract_list_pii_hashing_mapper import ListPIIHashingMapper @@ -20,7 +20,7 @@ class DVUserListPIIHashingMapper(ListPIIHashingMapper): def __init__(self): - self.logger = logging.getLogger("megalista.DVUserListPIIHashingMapper") + self.logger = logging.get_logger("megalista.DVUserListPIIHashingMapper") def _hash_user(self, user, hasher): hashable_keys = self._get_default_hasheable_keys() @@ -35,7 +35,7 @@ def _hash_user(self, user, hasher): processed_email = self.normalize_email(user["email"]) processed_user["hashedEmails"] = hasher.hash_field( processed_email) - except: + except Exception: self.logger.error(f"Error hashing email for user: {str(user)}") try: @@ -52,14 +52,14 @@ def _hash_user(self, user, hasher): processed_user["countryCode"] = user["mailing_address_country"] processed_user["zipCodes"] = user["mailing_address_zip"] - except: + except Exception: self.logger.error(f"Error hashing address for user: {str(user)}") try: if self._is_data_present(user, "phone"): processed_user["hashedPhoneNumbers"] = hasher.hash_field( user["phone"]) - except: + except Exception: self.logger.error(f"Error hashing phone for user: {str(user)}") if self._is_data_present(user, "mobile_device_id"): diff --git a/megalista_dataflow/mappers/dv_user_list_pii_hashing_mapper_test.py b/megalista_dataflow/mappers/dv_user_list_pii_hashing_mapper_test.py index efba5be8..c271b71b 100644 --- a/megalista_dataflow/mappers/dv_user_list_pii_hashing_mapper_test.py +++ b/megalista_dataflow/mappers/dv_user_list_pii_hashing_mapper_test.py @@ -15,6 +15,7 @@ from mappers.dv_user_list_pii_hashing_mapper import DVUserListPIIHashingMapper from models.execution import Batch +PHONE_1 = "+551199999999" def test_get_should_hash_fields(): @@ -40,7 +41,7 @@ def test_pii_hashing(mocker): users = [{ "email": "john@doe.com", - "phone": "+551199999999", + "phone": PHONE_1, "mailing_address_first_name": "John ", "mailing_address_last_name": "Doe", "mailing_address_zip": "12345", @@ -84,11 +85,11 @@ def test_pii_hashing(mocker): }, { "email": "ca.us@gmail.com", - "phone": "+551199999999", + "phone": PHONE_1, }, { "email": "us.ca@doe.com", - "phone": "+551199999999", + "phone": PHONE_1, } ] diff --git a/megalista_dataflow/mappers/executions_grouped_by_source_mapper.py b/megalista_dataflow/mappers/executions_grouped_by_source_mapper.py index c86cb994..2069ca6e 100644 --- a/megalista_dataflow/mappers/executions_grouped_by_source_mapper.py +++ b/megalista_dataflow/mappers/executions_grouped_by_source_mapper.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging from models.execution import Batch from mappers.abstract_list_pii_hashing_mapper import ListPIIHashingMapper @@ -22,7 +22,7 @@ class ExecutionsGroupedBySourceMapper(): def __init__(self): - self.logger = logging.getLogger("megalista.ExecutionsGroupedBySourceMapper") + self.logger = logging.get_logger("megalista.ExecutionsGroupedBySourceMapper") def encapsulate(self, element): return ExecutionsGroupedBySource(element[0], element[1]) diff --git a/megalista_dataflow/models/execution.py b/megalista_dataflow/models/execution.py index 1dcb44c5..028c784c 100644 --- a/megalista_dataflow/models/execution.py +++ b/megalista_dataflow/models/execution.py @@ -14,7 +14,6 @@ from enum import Enum from typing import Dict, List, Union, Any -import logging from apache_beam.typehints.decorators import with_output_types class DestinationType(Enum): @@ -260,6 +259,10 @@ def __init__( self._account_config = account_config self._source = source self._destination = destination + self._total_records = 0 + self._failed_records = 0 + self._successful_records = 0 + @property def source(self) -> Source: @@ -273,6 +276,44 @@ def destination(self) -> Destination: def account_config(self) -> AccountConfig: return self._account_config + @property + def summary_of_records(self) -> dict: + return { + 'total': self.total_records, + 'successful': self.successful_records, + 'failed': self._failed_records + } + + @property + def total_records(self): + return self._total_records + + @total_records.setter + def total_records(self, total_records): + self._total_records = total_records + + @property + def successful_records(self): + return self._successful_records + + @successful_records.setter + def successful_records(self, value): + self._successful_records = value + + def add_successful_record(self, _): + self._successful_records = self._successful_records + 1 + + @property + def failed_records(self): + return self._failed_records + + @failed_records.setter + def failed_records(self, value): + self._failed_records = value + + def add_failed_record(self, _): + self._failed_records = self._failed_records + 1 + def to_dict(self): return { 'account_config': self.account_config.to_dict(), diff --git a/megalista_dataflow/models/json_config.py b/megalista_dataflow/models/json_config.py index a24defb0..7753a535 100644 --- a/megalista_dataflow/models/json_config.py +++ b/megalista_dataflow/models/json_config.py @@ -22,9 +22,12 @@ class JsonConfig: def __init__(self, dataflow_options: DataflowOptions): self._dataflow_options = dataflow_options + def _get_json(self, url): + file_provider = FileProvider(url, self._dataflow_options, SourceType.FILE, "File Config (JSON)", False) + return file_provider.read().decode('utf-8') + def parse_json_from_url(self, url): - fileProvider = FileProvider(url, self._dataflow_options, SourceType.FILE, "File Config (JSON)", False) - data = json.loads(fileProvider.read().decode('utf-8')) + data = json.loads(self._get_json(url)) return data def get_value(self, config_json, key): diff --git a/megalista_dataflow/models/oauth_credentials_test.py b/megalista_dataflow/models/oauth_credentials_test.py index 9f1ef17e..221a3ce8 100644 --- a/megalista_dataflow/models/oauth_credentials_test.py +++ b/megalista_dataflow/models/oauth_credentials_test.py @@ -17,11 +17,11 @@ def test_init(): - id = StaticValueProvider(str, "id") + _id = StaticValueProvider(str, "id") secret = StaticValueProvider(str, "secret") access = StaticValueProvider(str, "access") refresh = StaticValueProvider(str, "refresh") - credentials = OAuthCredentials(id, secret, access, refresh) + credentials = OAuthCredentials(_id, secret, access, refresh) assert credentials.get_client_id() == "id" assert credentials.get_client_secret() == "secret" assert credentials.get_access_token() == "access" diff --git a/megalista_dataflow/models/sheets_config.py b/megalista_dataflow/models/sheets_config.py index fdfe5ade..cc9f4788 100644 --- a/megalista_dataflow/models/sheets_config.py +++ b/megalista_dataflow/models/sheets_config.py @@ -46,19 +46,17 @@ def get_range(self, sheet_id, range): def get_value(self, sheet_id, range): try: - range = self.get_range(sheet_id, range) + sheet_range = self.get_range(sheet_id, range) except Exception: return None - if range.get('values') is None: + if sheet_range.get('values') is None: return None - return range['values'][0][0] + return sheet_range['values'][0][0] def check_if_range_exists(self, sheet_id, range): try: self._get_sheets_service().spreadsheets().get(spreadsheetId=sheet_id, range=range).execute() - except: + except Exception: return False else: return True - finally: - pass diff --git a/megalista_dataflow/sources/batches_from_executions.py b/megalista_dataflow/sources/batches_from_executions.py index 983777f5..d95af98a 100644 --- a/megalista_dataflow/sources/batches_from_executions.py +++ b/megalista_dataflow/sources/batches_from_executions.py @@ -14,7 +14,7 @@ from enum import Enum import apache_beam as beam -import logging +from config import logging import json import functools @@ -34,12 +34,7 @@ _LOGGER_NAME = 'megalista.BatchesFromExecutions' - -def _convert_row_to_dict(row): - dict = {} - for key, value in row.items(): - dict[key] = value - return dict +_INT_MAX = 2147483647 class ExecutionCoder(coders.Coder): """A custom coder for the Execution class.""" @@ -52,7 +47,8 @@ def decode(self, s): def is_deterministic(self): return True - + + class ExecutionsGroupedBySourceCoder(coders.Coder): """A custom coder for the Execution class.""" @@ -78,6 +74,18 @@ def decode(self, s): def is_deterministic(self): return True + def estimate_size(self, o): + amount_of_rows = len(o.rows) + row_size = 0 + if amount_of_rows > 0: + row_size = len(json.dumps(o.rows[0]).encode('utf-8')) + estimate = amount_of_rows * row_size + # there is an overflow error if estimated size > _INT_MAX + if estimate > _INT_MAX: + estimate = _INT_MAX + return estimate + + class BatchesFromExecutions(beam.PTransform): """ @@ -104,6 +112,7 @@ def __init__(self, batch_size: int, error_handler: ErrorHandler): def process(self, grouped_elements): # grouped_elements[0] is the grouping key, the execution execution = grouped_elements[0] + execution.total_records = len(grouped_elements[1]) batch: List[Any] = [] # Keeps track of the batch iteration iteration = 1 @@ -117,9 +126,13 @@ def process(self, grouped_elements): yield Batch(execution, batch, iteration) class _BreakIntoExecutions(beam.DoFn): + def __init__(self, destination_type: DestinationType): + self._destination_type = destination_type + def process(self, el): for item in el: - yield item + if item[0].destination.destination_type == self._destination_type: + yield item def __init__( self, @@ -149,6 +162,6 @@ def expand(self, executions): ) | beam.ParDo(self._ReadDataSource(self._transactional_type, self._dataflow_options, self._error_handler)) | beam.Map(lambda el: [(execution, el.rows) for execution in iter(el.executions.executions)]) - | beam.ParDo(self._BreakIntoExecutions()) + | beam.ParDo(self._BreakIntoExecutions(self._destination_type)) | beam.ParDo(self._BatchElements(self._batch_size, self._error_handler)) ) diff --git a/megalista_dataflow/sources/batches_from_executions_test.py b/megalista_dataflow/sources/batches_from_executions_test.py new file mode 100644 index 00000000..262ba58e --- /dev/null +++ b/megalista_dataflow/sources/batches_from_executions_test.py @@ -0,0 +1,75 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sources.batches_from_executions import BatchesFromExecutions, DataRowsGroupedBySourceCoder, _INT_MAX +from models.execution import AccountConfig, DataRow, DataRowsGroupedBySource, SourceType, DestinationType, TransactionalType, Execution, Source, Destination, ExecutionsGroupedBySource + +from typing import List +import pytest + +def _get_execution() -> Execution: + return Execution( + AccountConfig('', False, '', '', ''), + Source( + 'source_name', + SourceType.BIG_QUERY, + [] + ), + Destination( + 'destination_name', + DestinationType.ADS_CUSTOMER_MATCH_CONTACT_INFO_UPLOAD, + [] + ) + ) + +@pytest.fixture +def execution() -> Execution: + return _get_execution() + +@pytest.fixture +def executions_grouped_by_source() -> ExecutionsGroupedBySource: + return ExecutionsGroupedBySource( + 'source_name', + [_get_execution()] + ) + +@pytest.fixture +def data_rows_grouped_by_source_coder() -> DataRowsGroupedBySourceCoder: + return DataRowsGroupedBySourceCoder() + +def test_data_rows_grouped_by_source_estimate_size_zero(mocker, data_rows_grouped_by_source_coder: DataRowsGroupedBySourceCoder, executions_grouped_by_source: ExecutionsGroupedBySource): + data_rows: List[DataRow] = [] + o = DataRowsGroupedBySource(executions_grouped_by_source, data_rows) + assert data_rows_grouped_by_source_coder.estimate_size(o) == 0 + +def test_data_rows_grouped_by_source_estimate_size_overflow(mocker, data_rows_grouped_by_source_coder: DataRowsGroupedBySourceCoder, executions_grouped_by_source: ExecutionsGroupedBySource): + item: DataRow = DataRow({ + 'phone': '5ecdb1fcdba73c56fc682fceb87166537e7d3990cbefcadb31ee23fe0add6322' + }) + data_rows: List[DataRow] = [item for _ in range(100000000)] + + o = DataRowsGroupedBySource(executions_grouped_by_source, data_rows) + assert data_rows_grouped_by_source_coder.estimate_size(o) == _INT_MAX + +def test_batch_elements(mocker, execution): + item: DataRow = DataRow({ + 'phone': '5ecdb1fcdba73c56fc682fceb87166537e7d3990cbefcadb31ee23fe0add6322' + }) + data_rows: List[DataRow] = [item for _ in range(11)] + batch_elements = BatchesFromExecutions._BatchElements(2, None) + grouped_elements = (execution, data_rows) + amount_of_batches = 0 + for _ in batch_elements.process(grouped_elements): + amount_of_batches = amount_of_batches + 1 + assert amount_of_batches == 6 \ No newline at end of file diff --git a/megalista_dataflow/sources/firestore_execution_source.py b/megalista_dataflow/sources/firestore_execution_source.py index 068e3949..5011cdcd 100644 --- a/megalista_dataflow/sources/firestore_execution_source.py +++ b/megalista_dataflow/sources/firestore_execution_source.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import distutils.util -import logging +from config import logging from apache_beam.options.value_provider import ValueProvider @@ -22,6 +22,7 @@ from models.execution import Execution, AccountConfig from models.execution import Source, SourceType +LOGGER_NAME = "megalista.FirestoreExecutionSource" class FirestoreExecutionSource(BaseBoundedSource): """ @@ -48,7 +49,7 @@ def document_to_dict(doc): return doc_dict firestore_collection = self._setup_firestore_collection.get() - logging.getLogger("megalista.FirestoreExecutionSource").info(f"Loading Firestore collection {firestore_collection}...") + logging.get_logger(LOGGER_NAME).info(f"Loading Firestore collection {firestore_collection}...") db = firestore.Client() entries = db.collection(self._setup_firestore_collection.get()).where('active', '==', 'yes').stream() entries = [document_to_dict(doc) for doc in entries] @@ -56,7 +57,7 @@ def document_to_dict(doc): account_data = document_to_dict(db.collection(self._setup_firestore_collection.get()).document('account_config').get()) if not account_data: - raise Exception('Firestore collection is absent') + raise OSError('Firestore collection is absent') google_ads_id = account_data.get('google_ads_id', 'empty') mcc_trix = account_data.get('mcc_trix', 'FALSE') mcc = False if mcc_trix is None else bool(distutils.util.strtobool(mcc_trix)) @@ -65,18 +66,18 @@ def document_to_dict(doc): campaign_manager_profile_id = account_data.get('campaign_manager_profile_id', 'empty') account_config = AccountConfig(google_ads_id, mcc, google_analytics_account_id, campaign_manager_profile_id, app_id) - logging.getLogger("megalista.FirestoreExecutionSource").info(f"Loaded: {account_config}") + logging.get_logger(LOGGER_NAME).info(f"Loaded: {account_config}") sources = self._read_sources(entries) destinations = self._read_destination(entries) if entries: for entry in entries: if entry['active'].upper() == 'YES': - logging.getLogger("megalista.FirestoreExecutionSource").info( + logging.get_logger(LOGGER_NAME).info( f"Executing step Source:{sources[entry['source_name']].source_name} -> Destination:{destinations[entry['destination_name']].destination_name}") yield Execution(account_config, sources[entry['source_name']], destinations[entry['destination_name']]) else: - logging.getLogger("megalista.FirestoreExecutionSource").warn("No schedules found!") + logging.get_logger(LOGGER_NAME).warn("No schedules found!") def _read_sources(self, entries): sources = {} @@ -86,7 +87,7 @@ def _read_sources(self, entries): source = Source(entry['source_name'], SourceType[entry['source']], metadata) sources[source.source_name] = source else: - logging.getLogger("megalista.FirestoreExecutionSource").warn("No sources found!") + logging.get_logger(LOGGER_NAME).warn("No sources found!") return sources def _read_destination(self, entries): @@ -115,7 +116,7 @@ def create_metadata_list(entry): entry_type = entry['type'] metadata = metadata_list.get(entry_type, None) if not metadata: - raise Exception(f'Upload type not implemented: {entry_type}') + raise NotImplementedError(f'Upload type not implemented: {entry_type}') entry_metadata = [] for m in metadata: # metadata_padding stands for the N/A fields in Sheets, preserving list indexes @@ -124,7 +125,7 @@ def create_metadata_list(entry): elif m in entry: entry_metadata.append(entry[m]) else: - raise Exception(f'Missing field in Firestore document for {entry_type}: {m}') + raise ValueError(f'Missing field in Firestore document for {entry_type}: {m}') return entry_metadata @@ -134,5 +135,5 @@ def create_metadata_list(entry): destination = Destination(entry['destination_name'], DestinationType[entry['type']], create_metadata_list(entry)) destinations[destination.destination_name] = destination else: - logging.getLogger("megalista.FirestoreExecutionSource").warn("No destinations found!") + logging.get_logger(LOGGER_NAME).warn("No destinations found!") return destinations diff --git a/megalista_dataflow/sources/json_execution_source.py b/megalista_dataflow/sources/json_execution_source.py index e322c97b..2e2793d5 100644 --- a/megalista_dataflow/sources/json_execution_source.py +++ b/megalista_dataflow/sources/json_execution_source.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging from apache_beam.options.value_provider import ValueProvider @@ -22,6 +22,7 @@ from models.execution import Source, SourceType from models.json_config import JsonConfig +LOGGER_NAME = "megalista.JsonExecutionSource" class JsonExecutionSource(BaseBoundedSource): """ @@ -36,11 +37,11 @@ def __init__(self, json_config: JsonConfig, setup_json_url: ValueProvider): def _do_count(self): json_url = self._setup_json_url.get() json_data = self._json_config.parse_json_from_url(json_url) - return len(json_data) + return len(json_data['Connections']) def read(self, range_tracker): json_url = self._setup_json_url.get() - logging.getLogger("megalista.JsonExecutionSource").info(f"Loading configuration JSON {json_url}...") + logging.get_logger(LOGGER_NAME).info(f"Loading configuration JSON {json_url}...") json_data = self._json_config.parse_json_from_url(json_url) google_ads_id = self._json_config.get_value(json_data, "GoogleAdsAccountId") @@ -54,7 +55,7 @@ def read(self, range_tracker): campaign_manager_profile_id = self._json_config.get_value(json_data, "CampaignManagerAccountId") account_config = AccountConfig(google_ads_id, mcc, google_analytics_account_id, campaign_manager_profile_id, app_id) - logging.getLogger("megalista.JsonExecutionSource").info(f"Loaded: {account_config}") + logging.get_logger(LOGGER_NAME).info(f"Loaded: {account_config}") sources = self._read_sources(self._json_config, json_data) destinations = self._read_destination(self._json_config, json_data) @@ -63,11 +64,11 @@ def read(self, range_tracker): if schedules: for schedule in schedules: if schedule["Enabled"]: - logging.getLogger("megalista.JsonExecutionSource").info( + logging.get_logger(LOGGER_NAME).info( f"Executing step Source:{schedule['Source']} -> Destination:{schedule['Destination']}") yield Execution(account_config, sources[schedule["Source"]], destinations[schedule["Destination"]]) else: - logging.getLogger("megalista.JsonExecutionSource").warn("No schedules found!") + logging.get_logger(LOGGER_NAME).warn("No schedules found!") @staticmethod def _read_sources(json_config, json_data): @@ -80,7 +81,7 @@ def _read_sources(json_config, json_data): [row["Dataset"], row["Table"]]) sources[source.source_name] = source else: - logging.getLogger("megalista.JsonExecutionSource").warn("No sources found!") + logging.get_logger(LOGGER_NAME).warn("No sources found!") return sources @staticmethod @@ -89,10 +90,11 @@ def _read_destination(json_config, json_data): destinations = {} if destinations_list: for row in destinations_list: + print(row) # Create Destinations using Name, Type, and Metadata destination = Destination(row["Name"], DestinationType[row["Type"]], row["Metadata"]) destinations[destination.destination_name] = destination else: - logging.getLogger("megalista.JsonExecutionSource").warn("No destinations found!") + logging.get_logger(LOGGER_NAME).warn("No destinations found!") return destinations diff --git a/megalista_dataflow/sources/json_execution_source_test.py b/megalista_dataflow/sources/json_execution_source_test.py new file mode 100644 index 00000000..5b58059c --- /dev/null +++ b/megalista_dataflow/sources/json_execution_source_test.py @@ -0,0 +1,69 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import MagicMock +from models.options import DataflowOptions +from models.json_config import JsonConfig +from sources.json_execution_source import JsonExecutionSource +from models.execution import AccountConfig, DataRow, DataRowsGroupedBySource, SourceType, DestinationType, TransactionalType, Execution, Source, Destination, ExecutionsGroupedBySource +from apache_beam.options.value_provider import StaticValueProvider + + +from typing import List, Dict +import pytest +from pytest_mock import MockFixture + +_JSON = '{"GoogleAdsAccountId":"","GoogleAdsMCC":false,"AppId":"","GoogleAnalyticsAccountId":"","CampaignManagerAccountId":"","Sources":[{"Name":"[BQ] Contact Info","Type":"BIG_QUERY","Dataset":"megalista_data","Table":"customer_match_contact_info"},{"Name":"[BQ] Contact Info - Email","Type":"BIG_QUERY","Dataset":"megalista_data","Table":"customer_match_contact_info_email"},{"Name":"[BQ] Contact Info - Phone","Type":"BIG_QUERY","Dataset":"megalista_data","Table":"customer_match_contact_info_phone"},{"Name":"[BQ] Contact Info - Mailing Address","Type":"BIG_QUERY","Dataset":"megalista_data","Table":"customer_match_contact_info_mailing_address"},{"Name":"[BQ] Ads - Offline Conversion (click)","Type":"BIG_QUERY","Dataset":"megalista_data","Table":"ads_offline_conversion"},{"Name":"[BQ - Carga] Contact Info 10m","Type":"BIG_QUERY","Dataset":"megalista_data","Table":"customer_match_contact_info_teste_carga_1"}],"Destinations":[{"Name":"[BQ] Contact Info","Type":"ADS_CUSTOMER_MATCH_CONTACT_INFO_UPLOAD","Metadata":["Megalista - Testing - Contact Info","ADD","10"]},{"Name":"[BQ] Contact Info - Email","Type":"ADS_CUSTOMER_MATCH_CONTACT_INFO_UPLOAD","Metadata":["Megalista - Testing - Contact Info (email)","ADD","10"]},{"Name":"[BQ] Contact Info - Phone","Type":"ADS_CUSTOMER_MATCH_CONTACT_INFO_UPLOAD","Metadata":["Megalista - Testing - Contact Info (phone)","ADD","10"]},{"Name":"[BQ] Contact Info - Mailing Address","Type":"ADS_CUSTOMER_MATCH_CONTACT_INFO_UPLOAD","Metadata":["Megalista - Testing - Contact Info (mailing address)","ADD","10"]},{"Name":"[BQ] Ads - Offline Conversion (click)","Type":"ADS_OFFLINE_CONVERSION","Metadata":["Megalista - testing - Offline Conversions (click)"]},{"Name":"[BQ] DV360 Contact Info","Type":"DV_CUSTOMER_MATCH_CONTACT_INFO_UPLOAD","Metadata":["633801967","Megalista - Testing - DV360 - Contact Info"]}],"Connections":[{"Enabled":false,"Source":"[BQ] Contact Info","Destination":"[BQ] Contact Info"},{"Enabled":false,"Source":"[BQ] Contact Info - Email","Destination":"[BQ] Contact Info - Email"},{"Enabled":false,"Source":"[BQ] Contact Info - Phone","Destination":"[BQ] Contact Info - Phone"},{"Enabled":false,"Source":"[BQ] Contact Info - Mailing Address","Destination":"[BQ] Contact Info - Mailing Address"},{"Enabled":false,"Source":"[BQ] Ads - Offline Conversion (click)","Destination":"[BQ] Ads - Offline Conversion (click)"},{"Enabled":false,"Source":"[BQ] Contact Info","Destination":"[BQ] DV360 Contact Info"}]}' + +@pytest.fixture +def json_config(mocker: MockFixture): + json_config = JsonConfig(DataflowOptions()) + return json_config + +def patch_json_config(mocker, json_config): + mocker.patch.object(json_config, '_get_json') + json_config._get_json.return_value = _JSON + + +def test_read_destinations(mocker, json_config): + patch_json_config(mocker, json_config) + json_data = json_config.parse_json_from_url('') + + destinations: Dict[str, Destination] = JsonExecutionSource._read_destination(json_config, json_data) + + key = '[BQ] Contact Info' + assert len(destinations) == 6 + assert destinations[key].destination_name == key + assert destinations[key].destination_type == DestinationType.ADS_CUSTOMER_MATCH_CONTACT_INFO_UPLOAD + assert destinations[key].destination_metadata == ['Megalista - Testing - Contact Info', 'ADD', '10'] + +def test_read_sources(mocker, json_config): + patch_json_config(mocker, json_config) + + json_data = json_config.parse_json_from_url('') + + sources: Dict[str, Source] = JsonExecutionSource._read_sources(json_config, json_data) + + key = '[BQ] Contact Info 2' + assert len(sources) == 6 + assert sources[key].source_name == key + assert sources[key].source_type == SourceType.BIG_QUERY + assert sources[key].source_metadata == ['megalista_data', 'customer_match_contact_info'] + +def test_count(mocker, json_config): + patch_json_config(mocker, json_config) + + execution_source: JsonExecutionSource = JsonExecutionSource(json_config, StaticValueProvider(str, '')) + + assert execution_source.count() == 6 \ No newline at end of file diff --git a/megalista_dataflow/sources/primary_execution_source.py b/megalista_dataflow/sources/primary_execution_source.py index ef140324..61cc6b84 100644 --- a/megalista_dataflow/sources/primary_execution_source.py +++ b/megalista_dataflow/sources/primary_execution_source.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging from apache_beam.options.value_provider import ValueProvider @@ -54,13 +54,13 @@ def __init__(self, def _do_count(self): if self._setup_sheet_id.get(): - logging.getLogger("megalista").info("Using Sheets count") + logging.get_logger("megalista").info("Using Sheets count") return self._sheets_execution_source._do_count() elif self._setup_firestore_collection.get(): - logging.getLogger("megalista").info("Using Firestore count") + logging.get_logger("megalista").info("Using Firestore count") return self._firestore_execution_source._do_count() elif self._setup_json_url.get(): - logging.getLogger("megalista").info("Using JSON count") + logging.get_logger("megalista").info("Using JSON count") return self._json_execution_source._do_count() else: raise ValueError("Configuration file/sheet not informed. Please check and run Megalista again.") @@ -69,15 +69,15 @@ def read(self, range_tracker): # config logging. has to be done inside a processing step due to Dataflow, since it removes # all log handlers in the beginning of the pipeline LoggingConfig.config_logging(self._show_code_lines_in_log.get()) - logging.getLogger("megalista").info(f"MEGALISTA build {MEGALISTA_VERSION}: Init.") + logging.get_logger("megalista").info(f"MEGALISTA build {MEGALISTA_VERSION}: Init.") if self._setup_sheet_id.get(): - logging.getLogger("megalista").info("Reading Sheets configuration") + logging.get_logger("megalista").info("Reading Sheets configuration") return self._sheets_execution_source.read(range_tracker) elif self._setup_firestore_collection.get(): - logging.getLogger("megalista").info("Reading Firestore configuration") + logging.get_logger("megalista").info("Reading Firestore configuration") return self._firestore_execution_source.read(range_tracker) else: - logging.getLogger("megalista").info("Reading JSON configuration") + logging.get_logger("megalista").info("Reading JSON configuration") return self._json_execution_source.read(range_tracker) \ No newline at end of file diff --git a/megalista_dataflow/sources/spreadsheet_execution_source.py b/megalista_dataflow/sources/spreadsheet_execution_source.py index 5b485955..dbc43a09 100644 --- a/megalista_dataflow/sources/spreadsheet_execution_source.py +++ b/megalista_dataflow/sources/spreadsheet_execution_source.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import distutils.util -import logging +from config import logging from apache_beam.options.value_provider import ValueProvider @@ -22,6 +22,7 @@ from models.execution import Source, SourceType from models.sheets_config import SheetsConfig +LOGGER_NAME = "megalista.SpreadsheetExecutionSource" class SpreadsheetExecutionSource(BaseBoundedSource): """ @@ -43,7 +44,7 @@ def _do_count(self): def read(self, range_tracker): sheet_id = self._setup_sheet_id.get() - logging.getLogger("megalista.SpreadsheetExecutionSource").info(f"Loading configuration sheet {sheet_id}...") + logging.get_logger(LOGGER_NAME).info(f"Loading configuration sheet {sheet_id}...") google_ads_id = self._sheets_config.get_value(sheet_id, "GoogleAdsAccountId") mcc_trix = self._sheets_config.get_value(sheet_id, "GoogleAdsMCC") mcc = False if mcc_trix is None else bool(distutils.util.strtobool(mcc_trix)) @@ -56,7 +57,7 @@ def read(self, range_tracker): campaign_manager_profile_id = self._sheets_config.get_value(sheet_id, "CampaignManagerAccountId") account_config = AccountConfig(google_ads_id, mcc, google_analytics_account_id, campaign_manager_profile_id, app_id) - logging.getLogger("megalista.SpreadsheetExecutionSource").info(f"Loaded: {account_config}") + logging.get_logger(LOGGER_NAME).info(f"Loaded: {account_config}") sources = self._read_sources(self._sheets_config, sheet_id) destinations = self._read_destination(self._sheets_config, sheet_id) @@ -65,32 +66,32 @@ def read(self, range_tracker): if 'values' in schedules_range: for schedule in schedules_range['values']: if schedule[0] == 'YES': - logging.getLogger("megalista.SpreadsheetExecutionSource").info( + logging.get_logger(LOGGER_NAME).info( f"Executing step Source:{sources[schedule[1]].source_name} -> Destination:{destinations[schedule[2]].destination_name}") yield Execution(account_config, sources[schedule[1]], destinations[schedule[2]]) else: - logging.getLogger("megalista.SpreadsheetExecutionSource").warn("No schedules found!") + logging.get_logger(LOGGER_NAME).warn("No schedules found!") @staticmethod def _read_sources(sheets_config, sheet_id): - range = sheets_config.get_range(sheet_id, 'SourcesRange') + _range = sheets_config.get_range(sheet_id, 'SourcesRange') sources = {} - if 'values' in range: - for row in range['values']: + if 'values' in _range: + for row in _range['values']: source = Source(row[0], SourceType[row[1]], row[2:]) sources[source.source_name] = source else: - logging.getLogger("megalista.SpreadsheetExecutionSource").warn("No sources found!") + logging.get_logger(LOGGER_NAME).warn("No sources found!") return sources @staticmethod def _read_destination(sheets_config, sheet_id): - range = sheets_config.get_range(sheet_id, 'DestinationsRange') + _range = sheets_config.get_range(sheet_id, 'DestinationsRange') destinations = {} - if 'values' in range: - for row in range['values']: + if 'values' in _range: + for row in _range['values']: destination = Destination(row[0], DestinationType[row[1]], row[2:]) destinations[destination.destination_name] = destination else: - logging.getLogger("megalista.SpreadsheetExecutionSource").warn("No destinations found!") + logging.get_logger(LOGGER_NAME).warn("No destinations found!") return destinations diff --git a/megalista_dataflow/steps/last_step.py b/megalista_dataflow/steps/last_step.py index c572d1b5..ea0cb3cd 100644 --- a/megalista_dataflow/steps/last_step.py +++ b/megalista_dataflow/steps/last_step.py @@ -14,11 +14,11 @@ from distutils.log import Log import apache_beam as beam -import logging -from error.logging_handler import LoggingHandler +from config import logging from models.execution import Execution from .megalista_step import MegalistaStep from config.logging import LoggingConfig +from typing import List class LastStep(MegalistaStep): def expand(self, executions): @@ -52,11 +52,20 @@ def extract_output(self, accumulator): return accumulator class PrintResultsDoFn(beam.DoFn): - def process(self, executions): - logging_handler = LoggingConfig.get_logging_handler() - - if logging_handler is None: - logging.getLogger("megalista").info(f"Clould not find error interception handler. Skipping error intereception.") - else: - if logging_handler.has_errors: - logging.getLogger("megalista.LOG").error(f"SUMMARY OF ERRORS:\n{LoggingHandler.format_records(logging_handler.error_records)}") + def process(self, executions): + executions_results = [] + execution_counter = 1 + for key in executions: + execution = executions[key] + summary_of_records = execution.summary_of_records + msg = f"{execution_counter}. {key}:\n \ + - Type: {str(execution.destination.destination_type)[16:]}\n \ + - Total records: {summary_of_records['total']}\n \ + - Successful: {summary_of_records['successful']}\n \ + - Failed: {summary_of_records['failed']}\n" + executions_results.append(msg) + execution_counter = execution_counter + 1 + summary_msg = '\n'.join(executions_results) + logging.get_logger("megalista.LOG").info(f"SUMMARY OF RESULTS:\n{summary_msg}") + if logging.has_errors(): + logging.get_logger("megalista.LOG").error(f"SUMMARY OF ERRORS:\n{logging.get_formatted_error_list()}") diff --git a/megalista_dataflow/steps/megalista_step.py b/megalista_dataflow/steps/megalista_step.py index 1cc55b60..be510a7e 100644 --- a/megalista_dataflow/steps/megalista_step.py +++ b/megalista_dataflow/steps/megalista_step.py @@ -44,4 +44,4 @@ def params(self): return self._params def expand(self, executions): - pass \ No newline at end of file + raise NotImplementedError() \ No newline at end of file diff --git a/megalista_dataflow/steps/processing_steps.py b/megalista_dataflow/steps/processing_steps.py index 49015d1d..b07ef72a 100644 --- a/megalista_dataflow/steps/processing_steps.py +++ b/megalista_dataflow/steps/processing_steps.py @@ -14,9 +14,11 @@ import apache_beam as beam +from typing import List + from .megalista_step import MegalistaStep from sources.batches_from_executions import BatchesFromExecutions, TransactionalType -from models.execution import DestinationType +from models.execution import DestinationType, Execution from error.error_handling import ErrorHandler from uploaders.support.transactional_events_results_writer import TransactionalEventsResultsWriter @@ -44,9 +46,15 @@ from third_party import THIRD_PARTY_STEPS +from config import logging + ADS_CM_HASHER = AdsUserListPIIHashingMapper() DV_CM_HASHER = DVUserListPIIHashingMapper() +def inform_finish_of_execution_filter(el: Execution): + logging.get_logger('megalista.ProcessingSteps').info('Execution finished', execution=el) + return el + class GoogleAdsSSDStep(MegalistaStep): def expand(self, executions): return ( @@ -67,6 +75,8 @@ def expand(self, executions): ErrorHandler(DestinationType.ADS_SSD_UPLOAD, self.params.error_notifier) ) ) + | "Finish" + >> beam.Filter(inform_finish_of_execution_filter) ) @@ -80,7 +90,7 @@ def expand(self, executions): self.params.dataflow_options, DestinationType.ADS_CUSTOMER_MATCH_MOBILE_DEVICE_ID_UPLOAD ) - | "Hash Users - Google Ads Customer Match Contact Info" + | "Hash Users - Google Ads Customer Match Mobile Device Id" >> beam.Map(ADS_CM_HASHER.hash_users) | "Upload - Google Ads Customer Match Mobile Device Id" >> beam.ParDo( @@ -90,6 +100,8 @@ def expand(self, executions): ErrorHandler(DestinationType.ADS_CUSTOMER_MATCH_MOBILE_DEVICE_ID_UPLOAD, self.params.error_notifier) ) ) + | "Finish" + >> beam.Filter(inform_finish_of_execution_filter) ) @@ -113,6 +125,8 @@ def expand(self, executions): ErrorHandler(DestinationType.ADS_CUSTOMER_MATCH_CONTACT_INFO_UPLOAD, self.params.error_notifier) ) ) + | "Finish" + >> beam.Filter(inform_finish_of_execution_filter) ) @@ -126,7 +140,7 @@ def expand(self, executions): self.params.dataflow_options, DestinationType.ADS_CUSTOMER_MATCH_USER_ID_UPLOAD ) - | "Hash Users - Google Ads Customer Match Contact Info" + | "Hash Users - Google Ads Customer Match User Id" >> beam.Map(ADS_CM_HASHER.hash_users) | "Upload - Google Ads Customer User Device Id" >> beam.ParDo( @@ -136,6 +150,8 @@ def expand(self, executions): ErrorHandler(DestinationType.ADS_CUSTOMER_MATCH_USER_ID_UPLOAD, self.params.error_notifier) ) ) + | "Finish" + >> beam.Filter(inform_finish_of_execution_filter) ) @@ -162,7 +178,10 @@ def expand(self, executions): >> TransactionalEventsResultsWriter( self.params._dataflow_options, TransactionalType.GCLID_TIME, - ErrorHandler(DestinationType.ADS_OFFLINE_CONVERSION, self.params.error_notifier)) + ErrorHandler(DestinationType.ADS_OFFLINE_CONVERSION, self.params.error_notifier) + ) + | "Finish" + >> beam.Filter(inform_finish_of_execution_filter) ) @@ -189,7 +208,10 @@ def expand(self, executions): >> TransactionalEventsResultsWriter( self.params._dataflow_options, TransactionalType.GCLID_TIME, - ErrorHandler(DestinationType.ADS_OFFLINE_CONVERSION_CALLS, self.params.error_notifier)) + ErrorHandler(DestinationType.ADS_OFFLINE_CONVERSION_CALLS, self.params.error_notifier) + ) + | "Finish" + >> beam.Filter(inform_finish_of_execution_filter) ) @@ -207,7 +229,10 @@ def expand(self, executions): | "Upload - GA user list" >> beam.ParDo(GoogleAnalyticsUserListUploaderDoFn(self.params._oauth_credentials, ErrorHandler(DestinationType.GA_USER_LIST_UPLOAD, - self.params.error_notifier))) + self.params.error_notifier)) + ) + | "Finish" + >> beam.Filter(inform_finish_of_execution_filter) ) @@ -232,6 +257,8 @@ def expand(self, executions): ErrorHandler(DestinationType.GA_DATA_IMPORT, self.params.error_notifier)) ) + | "Finish" + >> beam.Filter(inform_finish_of_execution_filter) ) @@ -253,7 +280,10 @@ def expand(self, executions): >> TransactionalEventsResultsWriter( self.params._dataflow_options, TransactionalType.UUID, - ErrorHandler(DestinationType.GA_MEASUREMENT_PROTOCOL, self.params.error_notifier)) + ErrorHandler(DestinationType.GA_MEASUREMENT_PROTOCOL, self.params.error_notifier) + ) + | "Finish" + >> beam.Filter(inform_finish_of_execution_filter) ) @@ -275,7 +305,10 @@ def expand(self, executions): >> TransactionalEventsResultsWriter( self.params._dataflow_options, TransactionalType.UUID, - ErrorHandler(DestinationType.GA_4_MEASUREMENT_PROTOCOL, self.params.error_notifier)) + ErrorHandler(DestinationType.GA_4_MEASUREMENT_PROTOCOL, self.params.error_notifier) + ) + | "Finish" + >> beam.Filter(inform_finish_of_execution_filter) ) @@ -300,7 +333,10 @@ def expand(self, executions): >> TransactionalEventsResultsWriter( self.params._dataflow_options, TransactionalType.UUID, - ErrorHandler(DestinationType.CM_OFFLINE_CONVERSION, self.params.error_notifier)) + ErrorHandler(DestinationType.CM_OFFLINE_CONVERSION, self.params.error_notifier) + ) + | "Finish" + >> beam.Filter(inform_finish_of_execution_filter) ) class DisplayVideoCustomerMatchDeviceIdStep(MegalistaStep): @@ -323,6 +359,8 @@ def expand(self, executions): ErrorHandler(DestinationType.DV_CUSTOMER_MATCH_DEVICE_ID_UPLOAD, self.params.error_notifier) ) ) + | "Finish" + >> beam.Filter(inform_finish_of_execution_filter) ) @@ -346,6 +384,8 @@ def expand(self, executions): ErrorHandler(DestinationType.DV_CUSTOMER_MATCH_CONTACT_INFO_UPLOAD, self.params.error_notifier) ) ) + | "Finish" + >> beam.Filter(inform_finish_of_execution_filter) ) PROCESSING_STEPS = [ diff --git a/megalista_dataflow/steps/processing_steps_test.py b/megalista_dataflow/steps/processing_steps_test.py new file mode 100644 index 00000000..de6ec75e --- /dev/null +++ b/megalista_dataflow/steps/processing_steps_test.py @@ -0,0 +1,37 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import MagicMock +from steps.megalista_step import MegalistaStepParams +from steps.processing_steps import PROCESSING_STEPS, ProcessingStep +from third_party import THIRD_PARTY_STEPS + +from models.options import DataflowOptions +from models.json_config import JsonConfig +from sources.json_execution_source import JsonExecutionSource +from models.execution import AccountConfig, DataRow, DataRowsGroupedBySource, SourceType, DestinationType, TransactionalType, Execution, Source, Destination, ExecutionsGroupedBySource +from apache_beam.options.value_provider import StaticValueProvider + + +from typing import List, Dict +import pytest +from pytest_mock import MockFixture + +def processing_step_expand_test(): + params: MegalistaStepParams = MegalistaStepParams('', DataflowOptions(), None) + step: ProcessingStep = ProcessingStep(params) + executions: List[Execution] = list() + processing_results = step.expand(executions) + + assert len(processing_results) == len(PROCESSING_STEPS) + len(THIRD_PARTY_STEPS) \ No newline at end of file diff --git a/megalista_dataflow/third_party/uploaders/appsflyer/appsflyer_s2s_uploader_async.py b/megalista_dataflow/third_party/uploaders/appsflyer/appsflyer_s2s_uploader_async.py index 5bab489f..52cb21f0 100644 --- a/megalista_dataflow/third_party/uploaders/appsflyer/appsflyer_s2s_uploader_async.py +++ b/megalista_dataflow/third_party/uploaders/appsflyer/appsflyer_s2s_uploader_async.py @@ -13,7 +13,7 @@ # limitations under the License. import asyncio -import logging +from config import logging import time from datetime import datetime from typing import Any, List, Optional @@ -25,6 +25,7 @@ from uploaders import utils from uploaders.uploaders import MegalistaUploader +LOGGER_NAME = "megalista.AppsFlyerS2SUploader" class AppsFlyerS2SUploaderDoFn(MegalistaUploader): def __init__(self, dev_key, error_handler: ErrorHandler): @@ -34,10 +35,6 @@ def __init__(self, dev_key, error_handler: ErrorHandler): self.app_id: Optional[str] = None self.timeout = ClientTimeout(total=15) # 15 sec timeout - def start_bundle(self): - pass - - async def _prepare_and_send(self, session, row, success_elements): #prepare payload @@ -83,7 +80,7 @@ async def _send_http_request(self, session, payload, curr_retry): await asyncio.sleep(curr_retry) return await self._send_http_request(session, payload, curr_retry+1) else: - logging.getLogger("megalista.AppsFlyerS2SUploader").error( + logging.get_logger(LOGGER_NAME).error( f"Fail to send event. Response code: {response.status}, " f"reason: {response.reason}") #print(await response.text()) #uncomment to troubleshoot @@ -94,7 +91,7 @@ async def _send_http_request(self, session, payload, curr_retry): await asyncio.sleep(curr_retry) return await self._send_http_request(session, payload, curr_retry+1) else: - logging.getLogger("megalista.AppsFlyerS2SUploader").error('Error inserting event: ' + str(exc)) + logging.get_logger(LOGGER_NAME).error('Error inserting event: ' + str(exc)) return -1 @@ -118,7 +115,7 @@ def bind_key(self, payload, row, row_key, name): payload[name] = row[row_key] - @utils.safe_process(logger=logging.getLogger("megalista.AppsFlyerS2SUploader")) + @utils.safe_process(logger=logging.get_logger(LOGGER_NAME)) def process(self, batch: Batch, **kwargs): success_elements: List[Any] = [] start_datetime = datetime.now() @@ -129,7 +126,7 @@ def process(self, batch: Batch, **kwargs): #send all requests asyncronously loop = asyncio.new_event_loop() future = asyncio.ensure_future(self._async_request_runner(batch.elements, success_elements), loop = loop) - responses = loop.run_until_complete(future) + _ = loop.run_until_complete(future) #wait to avoid api trotle @@ -137,7 +134,11 @@ def process(self, batch: Batch, **kwargs): min_duration_sec = len(batch.elements)/500 #Using Rate limitation = 500 per sec if delta_sec < min_duration_sec: time.sleep(min_duration_sec - delta_sec) - logging.getLogger("megalista.AppsFlyerS2SUploader").info( - f"Successfully uploaded {len(success_elements)}/{len(batch.elements)} events.") + logging.get_logger(LOGGER_NAME).info( + f"Successfully uploaded {len(success_elements)}/{len(batch.elements)} events.", execution=execution) + + execution.successful_records = execution.successful_records + len(success_elements) + execution.failed_records = execution.failed_records + (len(batch.elements) - len(success_elements)) + yield Batch(execution, success_elements) diff --git a/megalista_dataflow/uploaders/campaign_manager/campaign_manager_conversion_uploader.py b/megalista_dataflow/uploaders/campaign_manager/campaign_manager_conversion_uploader.py index a1527c5c..c4b5b6aa 100644 --- a/megalista_dataflow/uploaders/campaign_manager/campaign_manager_conversion_uploader.py +++ b/megalista_dataflow/uploaders/campaign_manager/campaign_manager_conversion_uploader.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging import math import time @@ -47,9 +47,6 @@ def _get_dcm_service(self): return build('dfareporting', 'v3.5', credentials=credentials) - def start_bundle(self): - pass - @staticmethod def _assert_all_list_names_are_present(any_execution): destination = any_execution.destination.destination_metadata @@ -62,7 +59,7 @@ def _assert_all_list_names_are_present(any_execution): raise ValueError( f'Missing destination information. Received {str(destination)}') - @utils.safe_process(logger=logging.getLogger(_LOGGER_NAME)) + @utils.safe_process(logger=logging.get_logger(_LOGGER_NAME)) def process(self, batch: Batch, **kwargs): self._do_process(batch, time.time()) return [batch] @@ -90,41 +87,10 @@ def _do_upload_data( service = self._get_dcm_service() conversions = [] - logger = logging.getLogger(_LOGGER_NAME) + logger = logging.get_logger(_LOGGER_NAME) for conversion in rows: - to_upload = { - 'floodlightActivityId': floodlight_activity_id, - 'floodlightConfigurationId': floodlight_configuration_id, - 'ordinal': math.floor(timestamp * 10e5), - 'timestampMicros': math.floor(timestamp * 10e5) - } - - if 'gclid' in conversion and conversion['gclid']: - to_upload['gclid'] = conversion['gclid'] - elif 'encryptedUserId' in conversion and conversion['encryptedUserId']: - to_upload['encryptedUserId'] = conversion['encryptedUserId'] - elif 'mobileDeviceId' in conversion and conversion['mobileDeviceId']: - to_upload['mobileDeviceId'] = conversion['mobileDeviceId'] - elif 'matchId' in conversion and conversion['matchId']: - to_upload['matchId'] = conversion['matchId'] - - if 'value' in conversion: - to_upload['value'] = float(conversion['value']) - if 'quantity' in conversion: - to_upload['quantity'] = conversion['quantity'] - if 'customVariables' in conversion: - custom_variables = [] - for r in conversion['customVariables']: - cv = { - 'type': r['type'], - 'value': r['value'], - 'kind': 'dfareporting#customFloodlightVariable', - } - custom_variables.append(cv) - to_upload['customVariables'] = custom_variables - - if 'timestamp' in conversion: - to_upload['timestampMicros'] = utils.get_timestamp_micros(conversion['timestamp']) + + to_upload = self._create_body(conversion, floodlight_activity_id, floodlight_configuration_id, timestamp) conversions.append(to_upload) @@ -132,22 +98,70 @@ def _do_upload_data( 'conversions': conversions, } - logger.info(f'Conversions: \n{conversions}') + logger.info(f'Conversions: \n{conversions}', execution=execution) request = service.conversions().batchinsert( profileId=campaign_manager_profile_id, body=request_body) response = request.execute() + amount_of_success = 0 + amount_of_failures = 0 + if response['hasFailures']: - logger.error(f'Error(s) inserting conversions:\n{response}') + logger.error(f'Error(s) inserting conversions:\n{response}', execution=execution) conversions_status = response['status'] error_messages = [] for status in conversions_status: if 'errors' in status: + amount_of_failures = amount_of_failures + 1 for error in status['errors']: error_messages.append('[{}]: {}'.format(error['code'], error['message'])) + else: + amount_of_success = amount_of_success + 1 final_error_message = 'Errors from API:\n{}'.format('\n'.join(error_messages)) - logger.error(final_error_message) + logger.error(final_error_message, execution=execution) self._add_error(execution, final_error_message) + else: + amount_of_success = len(rows) + + execution.successful_records = execution.successful_records + amount_of_success + execution.failed_records = execution.failed_records + amount_of_failures + + def _create_body(self, conversion, floodlight_activity_id, floodlight_configuration_id, timestamp): + to_upload = { + 'floodlightActivityId': floodlight_activity_id, + 'floodlightConfigurationId': floodlight_configuration_id, + 'ordinal': math.floor(timestamp * 10e5), + 'timestampMicros': math.floor(timestamp * 10e5) + } + + if 'gclid' in conversion and conversion['gclid']: + to_upload['gclid'] = conversion['gclid'] + elif 'encryptedUserId' in conversion and conversion['encryptedUserId']: + to_upload['encryptedUserId'] = conversion['encryptedUserId'] + elif 'mobileDeviceId' in conversion and conversion['mobileDeviceId']: + to_upload['mobileDeviceId'] = conversion['mobileDeviceId'] + elif 'matchId' in conversion and conversion['matchId']: + to_upload['matchId'] = conversion['matchId'] + + if 'value' in conversion: + to_upload['value'] = float(conversion['value']) + if 'quantity' in conversion: + to_upload['quantity'] = conversion['quantity'] + if 'customVariables' in conversion: + custom_variables = [] + for r in conversion['customVariables']: + cv = { + 'type': r['type'], + 'value': r['value'], + 'kind': 'dfareporting#customFloodlightVariable', + } + custom_variables.append(cv) + to_upload['customVariables'] = custom_variables + + if 'timestamp' in conversion: + to_upload['timestampMicros'] = utils.get_timestamp_micros(conversion['timestamp']) + + return to_upload \ No newline at end of file diff --git a/megalista_dataflow/uploaders/display_video/customer_match/abstract_uploader.py b/megalista_dataflow/uploaders/display_video/customer_match/abstract_uploader.py index 22e7b376..383a123b 100644 --- a/megalista_dataflow/uploaders/display_video/customer_match/abstract_uploader.py +++ b/megalista_dataflow/uploaders/display_video/customer_match/abstract_uploader.py @@ -12,15 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging -from typing import Dict, Any, List, Optional +from config import logging +from shutil import ExecError +from typing import Dict, Any, List, Optional, Tuple from apache_beam.options.value_provider import StaticValueProvider from google.oauth2.credentials import Credentials from googleapiclient.discovery import build from error.error_handling import ErrorHandler -from models.execution import AccountConfig, Destination +from models.execution import AccountConfig, Destination, Execution from models.execution import Batch from models.execution import DestinationType from models.oauth_credentials import OAuthCredentials @@ -39,7 +40,7 @@ def __init__(self, oauth_credentials: OAuthCredentials, developer_token: StaticV self.oauth_credentials = oauth_credentials self.developer_token = developer_token self.active = True - self._user_list_id_cache: Dict[str, Dict[str, any]] = {} + self._user_list_id_cache: Dict[str, Dict[str, Any]] = dict() def _get_dv_service(self): credentials = Credentials( @@ -62,16 +63,13 @@ def _get_dv_service(self): def _get_dv_audience_service(self): return self._get_dv_service().firstAndThirdPartyAudiences() - def start_bundle(self): - pass - def finish_bundle(self): super().finish_bundle() def _create_list_if_it_does_not_exist(self, advertiser_id: str, list_name: str, - list_definition: Dict[str, Any]) -> Dict[str, Any]: + list_definition: Dict[str, Any]) -> Tuple[bool, Dict[str, Any]]: was_audience_created = False if self._user_list_id_cache.get(list_name) is None: was_audience_created, self._user_list_id_cache[list_name] = \ @@ -84,7 +82,7 @@ def _do_create_list_if_it_does_not_exist(self, advertiser_id: str, list_name: str, list_definition: Dict[str, Any] - ) -> str: + ) -> Tuple[bool, Dict[str, Any]]: was_audience_created = False found_audience = self._get_advertiser_audience_by_display_name( @@ -92,7 +90,7 @@ def _do_create_list_if_it_does_not_exist(self, if found_audience is None: # Create list - logging.getLogger(_DEFAULT_LOGGER).info( + logging.get_logger(_DEFAULT_LOGGER).info( '%s list does not exist, creating...', list_name) # Marks the newly created audience @@ -103,26 +101,26 @@ def _do_create_list_if_it_does_not_exist(self, body=list_definition ).execute() - logging.getLogger(_DEFAULT_LOGGER).info( + logging.get_logger(_DEFAULT_LOGGER).info( 'List %s created with resource name: %s', list_name, found_audience['displayName']) else: - logging.getLogger(_DEFAULT_LOGGER).info( + logging.get_logger(_DEFAULT_LOGGER).info( 'List found with name: %s [%s]', found_audience['displayName'], found_audience['firstAndThirdPartyAudienceId'] ) return was_audience_created, found_audience - def _get_advertiser_audience_by_display_name(self, advertiser_id: str, display_name: str) -> Optional[str]: + def _get_advertiser_audience_by_display_name(self, advertiser_id: str, display_name: str) -> Optional[Dict[str, Any]]: """ Gets the advertise's audience by the displayName """ response = self._get_dv_audience_service().list( advertiserId=advertiser_id, pageSize=1, - filter=f'displayName : {display_name}' + filter=f'displayName : "{display_name}"' ).execute() - + if response and response['firstAndThirdPartyAudiences']: - return response['firstAndThirdPartyAudiences'][0] + return dict(response['firstAndThirdPartyAudiences'][0]) else: return None @@ -154,12 +152,12 @@ def _get_list_name(self, account_config: AccountConfig, destination: Destination def get_filtered_rows(self, rows: List[Any], keys: List[str]) -> List[Dict[str, Any]]: return [{key: row.get(key) for key in keys if key in row} for row in rows] - @utils.safe_process(logger=logging.getLogger(_DEFAULT_LOGGER)) - def process(self, batch: Batch, **kwargs) -> None: + @utils.safe_process(logger=logging.get_logger(_DEFAULT_LOGGER)) + def process(self, batch: Batch, **kwargs) -> List[Execution]: if not self.active: - logging.getLogger(_DEFAULT_LOGGER).warning( + logging.get_logger(_DEFAULT_LOGGER).warning( 'Skipping upload to DV, parameters not configured.') - return + return [] execution = batch.execution @@ -205,18 +203,20 @@ def process(self, batch: Batch, **kwargs) -> None: body=updated_list_definition ).execute() + execution.successful_records = execution.successful_records + len(rows) + return [execution] def get_list_definition(self, account_config: AccountConfig, destination_metadata: List[str], list_to_add: List[Dict[str, Any]]) -> Dict[str, Any]: - pass + raise NotImplementedError() def get_update_list_definition(self, account_config: AccountConfig, destination_metadata: List[str], list_to_add: List[Dict[str, Any]]) -> Dict[str, Any]: - pass + raise NotImplementedError() def get_row_keys(self) -> List[str]: - pass + raise NotImplementedError() def get_action_type(self) -> DestinationType: - pass + raise NotImplementedError() diff --git a/megalista_dataflow/uploaders/display_video/customer_match/contact_info_uploader.py b/megalista_dataflow/uploaders/display_video/customer_match/contact_info_uploader.py index 1f76638d..c81b1059 100644 --- a/megalista_dataflow/uploaders/display_video/customer_match/contact_info_uploader.py +++ b/megalista_dataflow/uploaders/display_video/customer_match/contact_info_uploader.py @@ -13,7 +13,7 @@ # limitations under the License. import apache_beam as beam -import logging +from config import logging from typing import Dict, Any, List diff --git a/megalista_dataflow/uploaders/display_video/customer_match/contact_info_uploader_test.py b/megalista_dataflow/uploaders/display_video/customer_match/contact_info_uploader_test.py index abe85462..4d29b496 100644 --- a/megalista_dataflow/uploaders/display_video/customer_match/contact_info_uploader_test.py +++ b/megalista_dataflow/uploaders/display_video/customer_match/contact_info_uploader_test.py @@ -36,11 +36,11 @@ def error_notifier(mocker): def uploader(mocker, error_notifier): mocker.patch('googleapiclient.discovery.build') mocker.patch('google.ads.googleads.oauth2') - id = StaticValueProvider(str, 'id') + _id = StaticValueProvider(str, 'id') secret = StaticValueProvider(str, 'secret') access = StaticValueProvider(str, 'access') refresh = StaticValueProvider(str, 'refresh') - credentials = OAuthCredentials(id, secret, access, refresh) + credentials = OAuthCredentials(_id, secret, access, refresh) return DisplayVideoCustomerMatchContactInfoUploaderDoFn(credentials, StaticValueProvider( str, 'devtoken'), @@ -81,7 +81,7 @@ def test_upload_add_users(mocker, uploader, error_notifier): uploader._get_dv_audience_service.return_value.list.assert_called_once_with( advertiserId='advertiser_id', pageSize=1, - filter='displayName : list_name' + filter='displayName : "list_name"' ) test_create_resquest = { @@ -114,15 +114,18 @@ def test_upload_update_users(mocker, uploader, error_notifier): mocker.patch.object(uploader, '_get_dv_audience_service') - audience = MagicMock() - audience.firstAndThirdPartyAudienceId = 12345 - audience.displayName = 'list_name' + audience_list = { + 'firstAndThirdPartyAudiences': [ + { + 'firstAndThirdPartyAudienceId': 12345, + 'displayName': 'list_name' + } + ] + } - audience_list = MagicMock() - audience_list['firstAndThirdPartyAudiences'] = [audience] uploader._get_dv_audience_service.return_value.list.return_value.execute.return_value = audience_list uploader._get_dv_audience_service.return_value.editCustomerMatchMembers.return_value = MagicMock() - + destination = Destination( 'dest1', DestinationType.DV_CUSTOMER_MATCH_CONTACT_INFO_UPLOAD, @@ -145,7 +148,7 @@ def test_upload_update_users(mocker, uploader, error_notifier): uploader._get_dv_audience_service.return_value.list.assert_called_once_with( advertiserId='advertiser_id', pageSize=1, - filter='displayName : list_name' + filter='displayName : "list_name"' ) test_update_resquest = { diff --git a/megalista_dataflow/uploaders/display_video/customer_match/mobile_uploader.py b/megalista_dataflow/uploaders/display_video/customer_match/mobile_uploader.py index e42dd44e..e4483db8 100644 --- a/megalista_dataflow/uploaders/display_video/customer_match/mobile_uploader.py +++ b/megalista_dataflow/uploaders/display_video/customer_match/mobile_uploader.py @@ -14,9 +14,9 @@ from functools import reduce import apache_beam as beam -import logging +from config import logging -from typing import List, Dict, Any +from typing import Callable, List, Dict, Any from uploaders.display_video.customer_match.abstract_uploader import DisplayVideoCustomerMatchAbstractUploaderDoFn from uploaders import utils as utils @@ -50,13 +50,23 @@ def get_row_keys(self) -> List[str]: def get_action_type(self) -> DestinationType: return DestinationType.DV_CUSTOMER_MATCH_DEVICE_ID_UPLOAD - def get_simplified_list(self, list_to_add: List[Dict[str, Any]]) -> Dict[str,Any]: + @staticmethod + def _add_item_into_simplified_list(accumulator: list, item: Any) -> list: + mobile_device_ids = [] + if type(item['mobileDeviceIds']) is list: + mobile_device_ids = item['mobileDeviceIds'] + else: + mobile_device_ids = [item['mobileDeviceIds']] + return accumulator + mobile_device_ids + + def get_simplified_list(self, list_to_add: List[Dict[str, Any]]) -> List[str]: # Alls devices must be within the same array # a single (Str) device id may be passed, so could a list od devices # Therefore, normalizes it within a single, one dimension list + return list( reduce( - lambda accumulator, item: accumulator+(item['mobileDeviceIds'] if type(item['mobileDeviceIds']) is list else [item['mobileDeviceIds']]), + DisplayVideoCustomerMatchMobileUploaderDoFn._add_item_into_simplified_list, list_to_add, [] ) diff --git a/megalista_dataflow/uploaders/display_video/customer_match/mobile_uploader_test.py b/megalista_dataflow/uploaders/display_video/customer_match/mobile_uploader_test.py index 8fdd7901..25e81f02 100644 --- a/megalista_dataflow/uploaders/display_video/customer_match/mobile_uploader_test.py +++ b/megalista_dataflow/uploaders/display_video/customer_match/mobile_uploader_test.py @@ -36,11 +36,11 @@ def error_notifier(mocker): def uploader(mocker, error_notifier): mocker.patch('googleapiclient.discovery.build') mocker.patch('google.ads.googleads.oauth2') - id = StaticValueProvider(str, 'id') + _id = StaticValueProvider(str, 'id') secret = StaticValueProvider(str, 'secret') access = StaticValueProvider(str, 'access') refresh = StaticValueProvider(str, 'refresh') - credentials = OAuthCredentials(id, secret, access, refresh) + credentials = OAuthCredentials(_id, secret, access, refresh) return DisplayVideoCustomerMatchMobileUploaderDoFn(credentials, StaticValueProvider( str, 'devtoken'), @@ -87,7 +87,7 @@ def test_upload_add_users(mocker, uploader, error_notifier): uploader._get_dv_audience_service.return_value.list.assert_called_once_with( advertiserId='advertiser_id', pageSize=1, - filter='displayName : list_name' + filter='displayName : "list_name"' ) test_create_resquest = { @@ -115,16 +115,17 @@ def test_upload_add_users(mocker, uploader, error_notifier): assert not error_notifier.were_errors_sent def test_upload_update_users(mocker, uploader, error_notifier): - mocker.patch.object(uploader, '_get_dv_audience_service') - audience = MagicMock() - audience.firstAndThirdPartyAudienceId = 12345 - audience.displayName = 'list_name' - - audience_list = MagicMock() - audience_list.firstAndThirdPartyAudiences = [audience] - + audience_list = { + 'firstAndThirdPartyAudiences': [ + { + 'firstAndThirdPartyAudienceId': 12345, + 'displayName': 'list_name' + } + ] + } + uploader._get_dv_audience_service.return_value.list.return_value.execute.return_value = audience_list uploader._get_dv_audience_service.return_value.editCustomerMatchMembers.return_value = MagicMock() @@ -154,7 +155,7 @@ def test_upload_update_users(mocker, uploader, error_notifier): uploader._get_dv_audience_service.return_value.list.assert_called_once_with( advertiserId='advertiser_id', pageSize=1, - filter='displayName : list_name' + filter='displayName : "list_name"' ) test_update_resquest = { @@ -169,6 +170,8 @@ def test_upload_update_users(mocker, uploader, error_notifier): } } + + uploader._get_dv_audience_service.return_value.editCustomerMatchMembers.assert_any_call( firstAndThirdPartyAudienceId=audience_list['firstAndThirdPartyAudiences'][0]['firstAndThirdPartyAudienceId'], body=test_update_resquest diff --git a/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_calls_uploader.py b/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_calls_uploader.py index 0c8dd8fe..73cef35b 100644 --- a/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_calls_uploader.py +++ b/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_calls_uploader.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging from apache_beam.options.value_provider import ValueProvider @@ -52,9 +52,6 @@ def _get_oc_action_service(self, customer_id: str): self.developer_token.get(), customer_id) - def start_bundle(self): - pass - def _get_customer_id(self, account_config:AccountConfig, destination:Destination) -> str: """ If the customer_id is present on the destination, returns it, otherwise defaults to the account_config info. @@ -76,7 +73,7 @@ def _assert_conversion_name_is_present(execution: Execution): str(destination))) @utils.safe_process( - logger=logging.getLogger('megalista.GoogleAdsOfflineUploader')) + logger=logging.get_logger('megalista.GoogleAdsOfflineUploader')) def process(self, batch: Batch, **kwargs): execution = batch.execution self._assert_conversion_name_is_present(execution) @@ -92,8 +89,11 @@ def process(self, batch: Batch, **kwargs): customer_id, batch.elements) + batch_with_successful_gclids = GoogleAdsOfflineUploaderCallsDoFn._get_new_batch_with_successfully_uploaded_gclids(batch, response) + return [batch_with_successful_gclids] + def _do_upload(self, oc_service: Any, execution: Execution, conversion_resource_name: str, customer_id: str, rows: List[Dict[str, Union[str, Dict[str, str]]]]): - logging.getLogger(_DEFAULT_LOGGER).info(f'Uploading {len(rows)} offline conversions (calls) on {conversion_resource_name} to Google Ads.') + logging.get_logger(_DEFAULT_LOGGER).info(f'Uploading {len(rows)} offline conversions (calls) on {conversion_resource_name} to Google Ads.', execution=execution) conversions = [{ 'conversion_action': conversion_resource_name, 'caller_id': conversion['caller_id'], @@ -115,6 +115,8 @@ def _do_upload(self, oc_service: Any, execution: Execution, conversion_resource_ if error_message: self._add_error(execution, error_message) + utils.update_execution_counters(execution, rows, response) + return response def _get_resource_name(self, customer_id: str, name: str): @@ -124,4 +126,13 @@ def _get_resource_name(self, customer_id: str, name: str): for batch in response_query: for row in batch.results: return row.conversion_action.resource_name - raise Exception(f'Conversion "{name}" could not be found on account {customer_id}') + raise ValueError(f'Conversion "{name}" could not be found on account {customer_id}') + + @staticmethod + def _get_new_batch_with_successfully_uploaded_gclids(batch: Batch, response): + def caller_id_lambda(result): return result.caller_id + + successful_caller_ids = list(map(caller_id_lambda, filter(caller_id_lambda, response.results))) + successful_elements = list(filter(lambda element: element['caller_id'] in successful_caller_ids, batch.elements)) + + return Batch(batch.execution, successful_elements) \ No newline at end of file diff --git a/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_calls_uploader_test.py b/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_calls_uploader_test.py index 92d446d9..f83d66c1 100644 --- a/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_calls_uploader_test.py +++ b/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_calls_uploader_test.py @@ -30,6 +30,19 @@ _account_config = AccountConfig('123-45567-890', False, 'ga_account_id', '', '') +time1 = '2020-04-09T14:13:55.0005' +time1_result = '2020-04-09 14:13:55-03:00' + +call_time1 = '2020-04-08T14:13:55.0005' +call_time1_result = '2020-04-08 14:13:55-03:00' + +time2 = '2020-04-09T13:13:55.0005' +time2_result = '2020-04-09 13:13:55-03:00' + +call_time2 = '2020-04-08T13:13:55.0005' +call_time2_result = '2020-04-08 13:13:55-03:00' + +CALLER_ID = '+5511987654321' @pytest.fixture() def error_notifier(): @@ -78,29 +91,17 @@ def test_conversion_upload(mocker, uploader): source = Source('orig1', SourceType.BIG_QUERY, ['dt1', 'buyers']) execution = Execution(_account_config, source, destination) - time1 = '2020-04-09T14:13:55.0005' - time1_result = '2020-04-09 14:13:55-03:00' - - call_time1 = '2020-04-08T14:13:55.0005' - call_time1_result = '2020-04-08 14:13:55-03:00' - - time2 = '2020-04-09T13:13:55.0005' - time2_result = '2020-04-09 13:13:55-03:00' - - call_time2 = '2020-04-08T13:13:55.0005' - call_time2_result = '2020-04-08 13:13:55-03:00' - element1 = { 'time': time1, 'call_time': call_time1, 'amount': '123', - 'caller_id': '+5511987654321' + 'caller_id': CALLER_ID } element2 = { 'time': time2, 'call_time': call_time2, 'amount': '234', - 'caller_id': '+5511987654321' + 'caller_id': CALLER_ID } batch = Batch(execution, [element1, element2]) @@ -123,13 +124,13 @@ def test_conversion_upload(mocker, uploader): 'call_start_date_time': call_time1_result, 'conversion_date_time': time1_result, 'conversion_value': 123, - 'caller_id': '+5511987654321' + 'caller_id': CALLER_ID }, { 'conversion_action': conversion_resource_name, 'call_start_date_time': call_time2_result, 'conversion_date_time': time2_result, 'conversion_value': 234, - 'caller_id': '+5511987654321' + 'caller_id': CALLER_ID }] }) @@ -145,29 +146,17 @@ def test_upload_with_ads_account_override(mocker, uploader): source = Source('orig1', SourceType.BIG_QUERY, ['dt1', 'buyers']) execution = Execution(_account_config, source, destination) - time1 = '2020-04-09T14:13:55.0005' - time1_result = '2020-04-09 14:13:55-03:00' - - call_time1 = '2020-04-08T14:13:55.0005' - call_time1_result = '2020-04-08 14:13:55-03:00' - - time2 = '2020-04-09T13:13:55.0005' - time2_result = '2020-04-09 13:13:55-03:00' - - call_time2 = '2020-04-08T13:13:55.0005' - call_time2_result = '2020-04-08 13:13:55-03:00' - element1 = { 'time': time1, 'call_time': call_time1, 'amount': '123', - 'caller_id': '+5511987654321' + 'caller_id': CALLER_ID } element2 = { 'time': time2, 'call_time': call_time2, 'amount': '234', - 'caller_id': '+5511987654321' + 'caller_id': CALLER_ID } batch = Batch(execution, [element1, element2]) @@ -191,13 +180,13 @@ def test_upload_with_ads_account_override(mocker, uploader): 'call_start_date_time': call_time1_result, 'conversion_date_time': time1_result, 'conversion_value': 123, - 'caller_id': '+5511987654321' + 'caller_id': CALLER_ID }, { 'conversion_action': conversion_resource_name, 'call_start_date_time': call_time2_result, 'conversion_date_time': time2_result, 'conversion_value': 234, - 'caller_id': '+5511987654321' + 'caller_id': CALLER_ID }] }) @@ -213,16 +202,12 @@ def test_should_not_notify_errors_when_api_call_is_successful(mocker, uploader, 'dest1', DestinationType.ADS_OFFLINE_CONVERSION_CALLS, ['user_list']) source = Source('orig1', SourceType.BIG_QUERY, ['dt1', 'buyers']) execution = Execution(_account_config, source, destination) - - time1 = '2020-04-09T14:13:55.0005' - - call_time1 = '2020-04-08T14:13:55.0005' - + element1 = { 'time': time1, 'call_time': call_time1, 'amount': '123', - 'caller_id': '+5511987654321' + 'caller_id': CALLER_ID } batch = Batch(execution, [element1]) @@ -256,16 +241,12 @@ def test_error_notification(mocker, uploader, error_notifier): 'dest1', DestinationType.ADS_OFFLINE_CONVERSION_CALLS, ['user_list']) source = Source('orig1', SourceType.BIG_QUERY, ['dt1', 'buyers']) execution = Execution(_account_config, source, destination) - - time1 = '2020-04-09T14:13:55.0005' - - call_time1 = '2020-04-08T14:13:55.0005' element1 = { 'time': time1, 'call_time': call_time1, 'amount': '123', - 'caller_id': '+5511987654321' + 'caller_id': CALLER_ID } batch = Batch(execution, [element1]) @@ -282,86 +263,3 @@ def test_error_notification(mocker, uploader, error_notifier): assert error_notifier.were_errors_sent is True assert error_notifier.destination_type is DestinationType.ADS_OFFLINE_CONVERSION_CALLS assert error_notifier.errors_sent == {execution: f'Error on uploading offline conversions (calls): {error_message}.'} - - -# def test_conversion_upload_and_error_notification(mocker, uploader, error_notifier): -# """ -# Scenario where some gclids are uploaded but some give errors -# """ - -# # arrange -# conversion_resource_name = 'user_list_resouce' -# arrange_conversion_resource_name_api_call(mocker, uploader, conversion_resource_name) - -# mocker.patch.object(uploader, '_get_oc_service') -# conversion_name = 'user_list' -# destination = Destination( -# 'dest1', DestinationType.ADS_OFFLINE_CONVERSION, ['user_list']) -# source = Source('orig1', SourceType.BIG_QUERY, ['dt1', 'buyers']) -# execution = Execution(_account_config, source, destination) - -# time1 = '2020-04-09T14:13:55.0005' -# time1_result = '2020-04-09 14:13:55-03:00' - -# time2 = '2020-04-09T13:13:55.0005' -# time2_result = '2020-04-09 13:13:55-03:00' - -# element1 = { -# 'time': time1, -# 'amount': '123', -# 'gclid': '456' -# } -# element2 = { -# 'time': time2, -# 'amount': '234', -# 'gclid': '567' -# } -# batch = Batch(execution, [element1, element2]) - -# # gclid '456' returns as successful by the API, while gclid '567' does not. -# # in this scenario, it's expected that both are present in the API call, -# # but since gclid '567' is not returned as successful by the API, an error is sent through error_notifier - -# error_message = 'Offline Conversion uploading failures' - -# gclid_result_mock1 = MagicMock() -# gclid_result_mock1.gclid = '456' - -# upload_return_mock = MagicMock() -# upload_return_mock.results = [gclid_result_mock1] -# upload_return_mock.partial_failure_error.message = error_message -# uploader._get_oc_service.return_value.upload_click_conversions.return_value = upload_return_mock - -# # act -# successful_uploaded_gclids_batch = uploader.process(batch)[0] -# uploader.finish_bundle() - -# # assert -# assert len(successful_uploaded_gclids_batch.elements) == 1 -# assert successful_uploaded_gclids_batch.elements[0] == element1 - -# uploader._get_ads_service.return_value.search_stream.assert_called_once_with( -# customer_id='12345567890', -# query=f"SELECT conversion_action.resource_name FROM conversion_action WHERE conversion_action.name = '{conversion_name}'" -# ) - -# uploader._get_oc_service.return_value.upload_click_conversions.assert_called_once_with(request={ -# 'customer_id': '12345567890', -# 'partial_failure': True, -# 'validate_only': False, -# 'conversions': [{ -# 'conversion_action': conversion_resource_name, -# 'conversion_date_time': time1_result, -# 'conversion_value': 123, -# 'gclid': '456' -# }, { -# 'conversion_action': conversion_resource_name, -# 'conversion_date_time': time2_result, -# 'conversion_value': 234, -# 'gclid': '567' -# }] -# }) - -# assert error_notifier.were_errors_sent is True -# assert error_notifier.destination_type is DestinationType.ADS_OFFLINE_CONVERSION -# assert error_notifier.errors_sent == {execution: f'Error on uploading offline conversions: {error_message}.'} diff --git a/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_uploader.py b/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_uploader.py index c05e509e..12c8bb65 100644 --- a/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_uploader.py +++ b/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_uploader.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging from apache_beam.options.value_provider import ValueProvider @@ -45,9 +45,6 @@ def _get_oc_service(self, customer_id): self.developer_token.get(), customer_id) - def start_bundle(self): - pass - def _get_customer_id(self, account_config:AccountConfig, destination:Destination) -> str: """ If the customer_id is present on the destination, returns it, otherwise defaults to the account_config info. @@ -77,7 +74,7 @@ def _assert_conversion_name_is_present(execution: Execution): str(destination))) @utils.safe_process( - logger=logging.getLogger('megalista.GoogleAdsOfflineUploader')) + logger=logging.get_logger('megalista.GoogleAdsOfflineUploader')) def process(self, batch: Batch, **kwargs): execution = batch.execution self._assert_conversion_name_is_present(execution) @@ -99,12 +96,11 @@ def process(self, batch: Batch, **kwargs): customer_id, batch.elements) - batch_with_successful_gclids = self._get_new_batch_with_successfully_uploaded_gclids(batch, response) - if len(batch_with_successful_gclids.elements) > 0: - return [batch_with_successful_gclids] + batch_with_successful_gclids = GoogleAdsOfflineUploaderDoFn._get_new_batch_with_successfully_uploaded_gclids(batch, response) + return [batch_with_successful_gclids] def _do_upload(self, oc_service, execution, conversion_resource_name, customer_id, rows): - logging.getLogger(_DEFAULT_LOGGER).info(f'Uploading {len(rows)} offline conversions on {conversion_resource_name} to Google Ads.') + logging.get_logger(_DEFAULT_LOGGER).info(f'Uploading {len(rows)} offline conversions on {conversion_resource_name} to Google Ads.', execution=execution) conversions = [{ 'conversion_action': conversion_resource_name, 'conversion_date_time': utils.format_date(conversion['time']), @@ -125,6 +121,8 @@ def _do_upload(self, oc_service, execution, conversion_resource_name, customer_i if error_message: self._add_error(execution, error_message) + utils.update_execution_counters(execution, rows, response) + return response def _get_resource_name(self, ads_service, customer_id: str, name: str): @@ -133,7 +131,7 @@ def _get_resource_name(self, ads_service, customer_id: str, name: str): for batch in response_query: for row in batch.results: return row.conversion_action.resource_name - raise Exception(f'Conversion "{name}" could not be found on account {customer_id}') + raise ValueError(f'Conversion "{name}" could not be found on account {customer_id}') @staticmethod def _get_new_batch_with_successfully_uploaded_gclids(batch: Batch, response): diff --git a/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_uploader_test.py b/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_uploader_test.py index 7789600f..4a0afd69 100644 --- a/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_uploader_test.py +++ b/megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_uploader_test.py @@ -31,6 +31,12 @@ _account_config = AccountConfig('123-45567-890', False, 'ga_account_id', '', '') +time1 = '2020-04-09T14:13:55.0005' +time1_result = '2020-04-09 14:13:55-03:00' + +time2 = '2020-04-09T13:13:55.0005' +time2_result = '2020-04-09 13:13:55-03:00' + @pytest.fixture() def error_notifier(): return MockErrorNotifier() @@ -78,12 +84,6 @@ def test_conversion_upload(mocker, uploader): source = Source('orig1', SourceType.BIG_QUERY, ['dt1', 'buyers']) execution = Execution(_account_config, source, destination) - time1 = '2020-04-09T14:13:55.0005' - time1_result = '2020-04-09 14:13:55-03:00' - - time2 = '2020-04-09T13:13:55.0005' - time2_result = '2020-04-09 13:13:55-03:00' - element1 = { 'time': time1, 'amount': '123', @@ -149,12 +149,6 @@ def test_upload_with_ads_account_override(mocker, uploader): source = Source('orig1', SourceType.BIG_QUERY, ['dt1', 'buyers']) execution = Execution(_account_config, source, destination) - time1 = '2020-04-09T14:13:55.0005' - time1_result = '2020-04-09 14:13:55-03:00' - - time2 = '2020-04-09T13:13:55.0005' - time2_result = '2020-04-09 13:13:55-03:00' - batch = Batch(execution, [{ 'time': time1, 'amount': '123', @@ -214,8 +208,7 @@ def test_should_not_notify_errors_when_api_call_is_successful(mocker, uploader, source = Source('orig1', SourceType.BIG_QUERY, ['dt1', 'buyers']) execution = Execution(_account_config, source, destination) - time1 = '2020-04-09T14:13:55.0005' - + element1 = { 'time': time1, 'amount': '123', @@ -254,7 +247,6 @@ def test_error_notification(mocker, uploader, error_notifier): source = Source('orig1', SourceType.BIG_QUERY, ['dt1', 'buyers']) execution = Execution(_account_config, source, destination) - time1 = '2020-04-09T14:13:55.0005' element1 = { 'time': time1, @@ -294,12 +286,6 @@ def test_conversion_upload_and_error_notification(mocker, uploader, error_notifi source = Source('orig1', SourceType.BIG_QUERY, ['dt1', 'buyers']) execution = Execution(_account_config, source, destination) - time1 = '2020-04-09T14:13:55.0005' - time1_result = '2020-04-09 14:13:55-03:00' - - time2 = '2020-04-09T13:13:55.0005' - time2_result = '2020-04-09 13:13:55-03:00' - element1 = { 'time': time1, 'amount': '123', diff --git a/megalista_dataflow/uploaders/google_ads/conversions/google_ads_ssd_uploader.py b/megalista_dataflow/uploaders/google_ads/conversions/google_ads_ssd_uploader.py index 66251c8f..2aefb1c9 100644 --- a/megalista_dataflow/uploaders/google_ads/conversions/google_ads_ssd_uploader.py +++ b/megalista_dataflow/uploaders/google_ads/conversions/google_ads_ssd_uploader.py @@ -13,7 +13,7 @@ # limitations under the License. import datetime -import logging +from config import logging from error.error_handling import ErrorHandler from models.execution import AccountConfig, Batch, Destination, Execution @@ -44,7 +44,7 @@ def _assert_conversion_metadata_is_present(execution: Execution): f'Missing destination information. Received {len(metadata)} entry(ies)') @utils.safe_process( - logger=logging.getLogger('megalista.GoogleAdsSSDUploader')) + logger=logging.get_logger('megalista.GoogleAdsSSDUploader')) def process(self, batch: Batch, **kwargs): execution = batch.execution self._assert_conversion_metadata_is_present(execution) @@ -69,7 +69,7 @@ def process(self, batch: Batch, **kwargs): def _do_upload(self, execution, offline_user_data_job_service, customer_id, currency_code, conversion_action_resource_name, ssd_external_upload_id, rows): - logger = logging.getLogger('megalista.GoogleAdsSSDUploader') + logger = logging.get_logger('megalista.GoogleAdsSSDUploader') # Upload is divided into 3 parts: # 1. Creates Job @@ -114,6 +114,9 @@ def _do_upload(self, execution, offline_user_data_job_service, customer_id, curr if error_message: self._add_error(execution, error_message) + utils.update_execution_counters(execution, rows, data_insertion_response) + + # 3. Runs the Job offline_user_data_job_service.run_offline_user_data_job(resource_name = job_resource_name) diff --git a/megalista_dataflow/uploaders/google_ads/conversions/google_ads_ssd_uploader_test.py b/megalista_dataflow/uploaders/google_ads/conversions/google_ads_ssd_uploader_test.py index 5d3214c1..54b832ff 100644 --- a/megalista_dataflow/uploaders/google_ads/conversions/google_ads_ssd_uploader_test.py +++ b/megalista_dataflow/uploaders/google_ads/conversions/google_ads_ssd_uploader_test.py @@ -40,15 +40,21 @@ _time2 = '2020-04-09T13:13:55.0005' _time2_result = '2020-04-09 13:13:55-03:00' +EMAIL_1 = 'a@a.com' +EMAIL_2 = 'b@b.com' +CURRENCY_BRL = 'BRL' +AMOUNT_1 = '123' +AMOUNT_2 = '234' + @pytest.fixture def uploader(mocker): mocker.patch('google.ads.googleads.client.GoogleAdsClient') mocker.patch('google.ads.googleads.oauth2') - id = StaticValueProvider(str, 'id') + _id = StaticValueProvider(str, 'id') secret = StaticValueProvider(str, 'secret') access = StaticValueProvider(str, 'access') refresh = StaticValueProvider(str, 'refresh') - credentials = OAuthCredentials(id, secret, access, refresh) + credentials = OAuthCredentials(_id, secret, access, refresh) return GoogleAdsSSDUploaderDoFn(credentials, StaticValueProvider(str, 'devtoken'), ErrorHandler(DestinationType.ADS_SSD_UPLOAD, MockErrorNotifier())) @@ -72,13 +78,13 @@ def create_batch(account_config, currency_override, account_override): execution = Execution(account_config, source, destination) return Batch(execution, [{ - 'hashed_email': 'a@a.com', + 'hashed_email': EMAIL_1, 'time': _time1, - 'amount': '123' + 'amount': AMOUNT_1 }, { - 'hashed_email': 'b@b.com', + 'hashed_email': EMAIL_2, 'time': _time2, - 'amount': '234' + 'amount': AMOUNT_2 }]) def test_get_service(mocker, uploader): @@ -108,24 +114,24 @@ def test_conversion_upload(mocker, uploader, ssd_batch): 'operations': [{ 'create': { 'user_identifiers': [{ - 'hashed_email': 'a@a.com' + 'hashed_email': EMAIL_1 }], 'transaction_attribute': { 'conversion_action': conversion_name_resource_name, - 'currency_code': 'BRL', - 'transaction_amount_micros': '123', + 'currency_code': CURRENCY_BRL, + 'transaction_amount_micros': AMOUNT_1, 'transaction_date_time': _time1_result } } }, { 'create': { 'user_identifiers': [{ - 'hashed_email': 'b@b.com' + 'hashed_email': EMAIL_2 }], 'transaction_attribute': { 'conversion_action': conversion_name_resource_name, - 'currency_code': 'BRL', - 'transaction_amount_micros': '234', + 'currency_code': CURRENCY_BRL, + 'transaction_amount_micros': AMOUNT_2, 'transaction_date_time': _time2_result } } @@ -150,24 +156,24 @@ def test_conversion_upload_account_and_currency_override(mocker, uploader, ssd_b 'operations': [{ 'create': { 'user_identifiers': [{ - 'hashed_email': 'a@a.com' + 'hashed_email': EMAIL_1 }], 'transaction_attribute': { 'conversion_action': conversion_name_resource_name, 'currency_code': 'currency_override', - 'transaction_amount_micros': '123', + 'transaction_amount_micros': AMOUNT_1, 'transaction_date_time': _time1_result } } }, { 'create': { 'user_identifiers': [{ - 'hashed_email': 'b@b.com' + 'hashed_email': EMAIL_2 }], 'transaction_attribute': { 'conversion_action': conversion_name_resource_name, 'currency_code': 'currency_override', - 'transaction_amount_micros': '234', + 'transaction_amount_micros': AMOUNT_2, 'transaction_date_time': _time2_result } } @@ -192,24 +198,24 @@ def test_conversion_mcc_account_override(mocker, uploader, ssd_batch_with_mcc_ac 'operations': [{ 'create': { 'user_identifiers': [{ - 'hashed_email': 'a@a.com' + 'hashed_email': EMAIL_1 }], 'transaction_attribute': { 'conversion_action': conversion_name_resource_name, - 'currency_code': 'BRL', - 'transaction_amount_micros': '123', + 'currency_code': CURRENCY_BRL, + 'transaction_amount_micros': AMOUNT_1, 'transaction_date_time': _time1_result } } }, { 'create': { 'user_identifiers': [{ - 'hashed_email': 'b@b.com' + 'hashed_email': EMAIL_2 }], 'transaction_attribute': { 'conversion_action': conversion_name_resource_name, - 'currency_code': 'BRL', - 'transaction_amount_micros': '234', + 'currency_code': CURRENCY_BRL, + 'transaction_amount_micros': AMOUNT_2, 'transaction_date_time': _time2_result } } diff --git a/megalista_dataflow/uploaders/google_ads/customer_match/abstract_uploader.py b/megalista_dataflow/uploaders/google_ads/customer_match/abstract_uploader.py index d8cf080d..527b2859 100644 --- a/megalista_dataflow/uploaders/google_ads/customer_match/abstract_uploader.py +++ b/megalista_dataflow/uploaders/google_ads/customer_match/abstract_uploader.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging from typing import Dict, Any, List, Optional, Union from apache_beam.options.value_provider import StaticValueProvider @@ -25,9 +25,12 @@ from uploaders import utils from uploaders.google_ads import ADS_API_VERSION from uploaders.uploaders import MegalistaUploader +from google.api_core.exceptions import ResourceExhausted +from time import sleep _DEFAULT_LOGGER: str = 'megalista.GoogleAdsCustomerMatchAbstractUploader' - +_DEFAULT_TIMEOUT: int = 60 +_MAX_RETRIES: int = 6 class GoogleAdsCustomerMatchAbstractUploaderDoFn(MegalistaUploader): @@ -42,12 +45,9 @@ def __init__(self, oauth_credentials: OAuthCredentials, developer_token: StaticV self._user_list_id_cache: Dict[str, str] = {} self._job_cache: Dict[str, Dict[str, str]] = {} - def start_bundle(self): - pass - def finish_bundle(self): for job_definition in self._job_cache.values(): - logging.getLogger(_DEFAULT_LOGGER).info(f"Running job {job_definition['job_resource_name']}") + logging.get_logger(_DEFAULT_LOGGER).info(f"Running job {job_definition['job_resource_name']}") self._get_offline_user_data_job_service(job_definition['login_customer_id']).run_offline_user_data_job( resource_name=job_definition['job_resource_name']) @@ -78,7 +78,7 @@ def _do_create_list_if_it_does_not_exist(self, if resource_name is None: # Create list - logging.getLogger(_DEFAULT_LOGGER).info( + logging.get_logger(_DEFAULT_LOGGER).info( '%s list does not exist, creating...', list_name) request = { 'customer_id': customer_id, @@ -93,10 +93,10 @@ def _do_create_list_if_it_does_not_exist(self, user_list_service_response = user_list_service.mutate_user_lists(request) for result in user_list_service_response.results: resource_name = result.resource_name - logging.getLogger(_DEFAULT_LOGGER).info('List %s created with resource name: %s', + logging.get_logger(_DEFAULT_LOGGER).info('List %s created with resource name: %s', list_name, resource_name) else: - logging.getLogger(_DEFAULT_LOGGER).info('List %s found with resource name: %s', + logging.get_logger(_DEFAULT_LOGGER).info('List %s found with resource name: %s', list_name, resource_name) return str(resource_name) @@ -191,10 +191,10 @@ def _get_remove_all(self, operator: str) -> bool: def get_filtered_rows(self, rows: List[Any], keys: List[str]) -> List[Dict[str, Any]]: return [{key: row.get(key) for key in keys if key in row} for row in rows] - @utils.safe_process(logger=logging.getLogger(_DEFAULT_LOGGER)) + @utils.safe_process(logger=logging.get_logger(_DEFAULT_LOGGER)) def process(self, batch: Batch, **kwargs): if not self.active: - logging.getLogger(_DEFAULT_LOGGER).warning( + logging.get_logger(_DEFAULT_LOGGER).warning( 'Skipping upload to ads, parameters not configured.') return @@ -239,22 +239,42 @@ def process(self, batch: Batch, **kwargs): 'operations': operations } - data_insertion_response = offline_user_data_job_service.add_offline_user_data_job_operations( - request=data_insertion_payload) - + timeout_multiplier = 1 + data_insertion_response = None + retry = 0 + while data_insertion_response is None: + try: + data_insertion_response = offline_user_data_job_service.add_offline_user_data_job_operations( + request=data_insertion_payload) + except ResourceExhausted as e: + if retry < _MAX_RETRIES: + retry = retry + 1 + logging.get_logger(_DEFAULT_LOGGER).warning( + f'Resource Exhausted exception: {e}. Retrying... ({retry} of {_MAX_RETRIES})', execution=execution) + sleep(timeout_multiplier * _DEFAULT_TIMEOUT) + timeout_multiplier = timeout_multiplier * 2 + + else: + logging.get_logger(_DEFAULT_LOGGER).warning( + f'Resource Exhausted exception: {e}. No retries left.', execution=execution) + raise e + error_message = utils.print_partial_error_messages(_DEFAULT_LOGGER, 'uploading customer match', data_insertion_response) if error_message: self._add_error(execution, error_message) + utils.update_execution_counters(execution, batch.elements, data_insertion_response) + return [execution] def get_list_definition(self, account_config: AccountConfig, destination_metadata: List[str]) -> Dict[str, Any]: - pass + raise NotImplementedError() def get_row_keys(self) -> List[str]: - pass + raise NotImplementedError() def get_action_type(self) -> DestinationType: - pass + raise NotImplementedError() + diff --git a/megalista_dataflow/uploaders/google_ads/customer_match/contact_info_uploader.py b/megalista_dataflow/uploaders/google_ads/customer_match/contact_info_uploader.py index 939033ed..8bf41a53 100644 --- a/megalista_dataflow/uploaders/google_ads/customer_match/contact_info_uploader.py +++ b/megalista_dataflow/uploaders/google_ads/customer_match/contact_info_uploader.py @@ -13,7 +13,7 @@ # limitations under the License. import apache_beam as beam -import logging +from config import logging from typing import Dict, Any, List diff --git a/megalista_dataflow/uploaders/google_ads/customer_match/contact_info_uploader_test.py b/megalista_dataflow/uploaders/google_ads/customer_match/contact_info_uploader_test.py index d09c54db..63bb7808 100644 --- a/megalista_dataflow/uploaders/google_ads/customer_match/contact_info_uploader_test.py +++ b/megalista_dataflow/uploaders/google_ads/customer_match/contact_info_uploader_test.py @@ -34,11 +34,11 @@ def error_notifier(mocker): def uploader(mocker, error_notifier): mocker.patch('google.ads.googleads.client.GoogleAdsClient') mocker.patch('google.ads.googleads.oauth2') - id = StaticValueProvider(str, 'id') + _id = StaticValueProvider(str, 'id') secret = StaticValueProvider(str, 'secret') access = StaticValueProvider(str, 'access') refresh = StaticValueProvider(str, 'refresh') - credentials = OAuthCredentials(id, secret, access, refresh) + credentials = OAuthCredentials(_id, secret, access, refresh) return GoogleAdsCustomerMatchContactInfoUploaderDoFn(credentials, StaticValueProvider(str, 'devtoken'), ErrorHandler( DestinationType.ADS_CUSTOMER_MATCH_CONTACT_INFO_UPLOAD, error_notifier)) diff --git a/megalista_dataflow/uploaders/google_ads/customer_match/mobile_uploader.py b/megalista_dataflow/uploaders/google_ads/customer_match/mobile_uploader.py index e1600fc1..1596158f 100644 --- a/megalista_dataflow/uploaders/google_ads/customer_match/mobile_uploader.py +++ b/megalista_dataflow/uploaders/google_ads/customer_match/mobile_uploader.py @@ -13,7 +13,7 @@ # limitations under the License. import apache_beam as beam -import logging +from config import logging from typing import List, Dict, Any diff --git a/megalista_dataflow/uploaders/google_analytics/google_analytics_4_measurement_protocol.py b/megalista_dataflow/uploaders/google_analytics/google_analytics_4_measurement_protocol.py index 3b3df675..f3aa3362 100644 --- a/megalista_dataflow/uploaders/google_analytics/google_analytics_4_measurement_protocol.py +++ b/megalista_dataflow/uploaders/google_analytics/google_analytics_4_measurement_protocol.py @@ -14,25 +14,23 @@ import json -import logging -from typing import Dict, Any +from config import logging +from typing import Dict, Any, Tuple, Optional import requests from error.error_handling import ErrorHandler -from models.execution import Batch +from models.execution import Batch, Execution from uploaders import utils from uploaders.uploaders import MegalistaUploader +LOGGER_NAME = 'megalista.GoogleAnalytics4MeasurementProtocolUploader' class GoogleAnalytics4MeasurementProtocolUploaderDoFn(MegalistaUploader): def __init__(self, error_handler: ErrorHandler): super().__init__(error_handler) self.API_URL = 'https://www.google-analytics.com/mp/collect' - def start_bundle(self): - pass - @staticmethod def _str2bool(s: str) -> bool: return s.lower() == 'true' @@ -41,7 +39,7 @@ def _str2bool(s: str) -> bool: def _exactly_one_of(a: Any, b: Any) -> bool: return (a and not b) or (not a and b) - @utils.safe_process(logger=logging.getLogger('megalista.GoogleAnalytics4MeasurementProtocolUploader')) + @utils.safe_process(logger=logging.get_logger(LOGGER_NAME)) def process(self, batch: Batch, **kwargs): return self.do_process(batch) @@ -52,27 +50,9 @@ def do_process(self, batch: Batch): api_secret = execution.destination.destination_metadata[0] is_event = self._str2bool(execution.destination.destination_metadata[1]) is_user_property = self._str2bool(execution.destination.destination_metadata[2]) - non_personalized_ads = self._str2bool(execution.destination.destination_metadata[3]) - - firebase_app_id = None - if len(execution.destination.destination_metadata) >= 5: - firebase_app_id = execution.destination.destination_metadata[4] - - measurement_id = None - if len(execution.destination.destination_metadata) >= 6: - measurement_id = execution.destination.destination_metadata[5] - - if not self._exactly_one_of(firebase_app_id, measurement_id): - raise ValueError( - 'GA4 MP should be called either with a firebase_app_id (for apps) or a measurement_id (for web)') - - if not self._exactly_one_of(is_event, is_user_property): - raise ValueError( - 'GA4 MP should be called either for sending events or a user properties') + non_personalized_ads = self._str2bool(execution.destination.destination_metadata[3]) - payload: Dict[str, Any] = { - 'nonPersonalizedAds': non_personalized_ads - } + firebase_app_id, measurement_id = self._getIds(execution, is_event, is_user_property) accepted_elements = [] @@ -80,49 +60,93 @@ def do_process(self, batch: Batch): app_instance_id = row.get('app_instance_id') client_id = row.get('client_id') user_id = row.get('user_id') - if 'timestamp_micros' in row: - payload['timestamp_micros'] = int(str(row.get('timestamp_micros'))) - + if not self._exactly_one_of(app_instance_id, client_id): raise ValueError( 'GA4 MP should be called either with an app_instance_id (for apps) or a client_id (for web)') - - if is_event: - params = {k: v for k, v in row.items() if k not in ('name', 'app_instance_id', 'client_id', 'uuid', 'user_id', 'timestamp_micros') and v is not None} - payload['events'] = [{'name': row['name'], 'params': params}] - if is_user_property: - payload['userProperties'] = {k: {'value': v} for k, v in row.items() if k not in ('app_instance_id', 'client_id', 'uuid', 'user_id', 'timestamp_micros') and v is not None} - payload['events'] = {'name': 'user_property_addition_event', 'params': {}} + if firebase_app_id and not app_instance_id: + raise ValueError( + 'GA4 MP needs an app_instance_id parameter when used for an App Stream.') - url_container = [f'{self.API_URL}?api_secret={api_secret}'] + if measurement_id and not client_id: + raise ValueError( + 'GA4 MP needs a client_id parameter when used for a Web Stream.') - if firebase_app_id: - url_container.append(f'&firebase_app_id={firebase_app_id}') - if not app_instance_id: - raise ValueError( - 'GA4 MP needs an app_instance_id parameter when used for an App Stream.') - payload['app_instance_id'] = app_instance_id - - if measurement_id: - url_container.append(f'&measurement_id={measurement_id}') - if not client_id: - raise ValueError( - 'GA4 MP needs a client_id parameter when used for a Web Stream.') - payload['client_id'] = client_id + + url = self._getUrl(api_secret, firebase_app_id, measurement_id) - if user_id: - payload['user_id'] = user_id + payload = self._fillPayload(row, non_personalized_ads, is_event, is_user_property, firebase_app_id, app_instance_id, measurement_id, client_id, user_id) - url = ''.join(url_container) response = requests.post(url,data=json.dumps(payload)) if response.status_code != 204: error_message = f'Error calling GA4 MP {response.status_code}: {str(response.content)}' - logging.getLogger('megalista.GoogleAnalytics4MeasurementProtocolUploader').error(error_message) + logging.get_logger(LOGGER_NAME).error(error_message, execution=execution) self._add_error(execution, error_message) else: accepted_elements.append(row) - logging.getLogger('megalista.GoogleAnalytics4MeasurementProtocolUploader').info( - f'Successfully uploaded {len(accepted_elements)}/{len(batch.elements)} events.') + logging.get_logger(LOGGER_NAME).info( + f'Successfully uploaded {len(accepted_elements)}/{len(batch.elements)} events.', execution=execution) + + execution.successful_records = execution.successful_records + len(accepted_elements) + execution.failed_records = execution.failed_records + (len(batch.elements) - len(accepted_elements)) + return [Batch(execution, accepted_elements)] + + def _getIds(self, execution: Execution, is_event: bool, is_user_property: bool) -> Tuple[Optional[str], Optional[str]]: + firebase_app_id = None + if len(execution.destination.destination_metadata) >= 5: + firebase_app_id = execution.destination.destination_metadata[4] + + measurement_id = None + if len(execution.destination.destination_metadata) >= 6: + measurement_id = execution.destination.destination_metadata[5] + + if not self._exactly_one_of(firebase_app_id, measurement_id): + raise ValueError( + 'GA4 MP should be called either with a firebase_app_id (for apps) or a measurement_id (for web)') + + if not self._exactly_one_of(is_event, is_user_property): + raise ValueError( + 'GA4 MP should be called either for sending events or a user properties') + + return firebase_app_id, measurement_id + + def _getUrl(self, api_secret, firebase_app_id, measurement_id): + url_container = [f'{self.API_URL}?api_secret={api_secret}'] + + if firebase_app_id: + url_container.append(f'&firebase_app_id={firebase_app_id}') + + if measurement_id: + url_container.append(f'&measurement_id={measurement_id}') + + return ''.join(url_container) + + def _fillPayload(self, row, non_personalized_ads, is_event, is_user_property, firebase_app_id, app_instance_id, measurement_id, client_id, user_id): + payload: Dict[str, Any] = { + 'nonPersonalizedAds': non_personalized_ads + } + + if 'timestamp_micros' in row: + payload['timestamp_micros'] = int(str(row.get('timestamp_micros'))) + + if is_event: + params = {k: v for k, v in row.items() if k not in ('name', 'app_instance_id', 'client_id', 'uuid', 'user_id', 'timestamp_micros') and v is not None} + payload['events'] = [{'name': row['name'], 'params': params}] + + if is_user_property: + payload['userProperties'] = {k: {'value': v} for k, v in row.items() if k not in ('app_instance_id', 'client_id', 'uuid', 'user_id', 'timestamp_micros') and v is not None} + payload['events'] = {'name': 'user_property_addition_event', 'params': {}} + + if firebase_app_id: + payload['app_instance_id'] = app_instance_id + + if measurement_id: + payload['client_id'] = client_id + + if user_id: + payload['user_id'] = user_id + + return payload diff --git a/megalista_dataflow/uploaders/google_analytics/google_analytics_data_import_eraser.py b/megalista_dataflow/uploaders/google_analytics/google_analytics_data_import_eraser.py index 288f636d..add56ee6 100644 --- a/megalista_dataflow/uploaders/google_analytics/google_analytics_data_import_eraser.py +++ b/megalista_dataflow/uploaders/google_analytics/google_analytics_data_import_eraser.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging from google.oauth2.credentials import Credentials from googleapiclient.discovery import build @@ -22,6 +22,7 @@ from uploaders import utils from uploaders.uploaders import MegalistaUploader +LOGGER_NAME = 'megalista.GoogleAnalyticsDataImportUploader' class GoogleAnalyticsDataImportEraser(MegalistaUploader): """ @@ -47,9 +48,6 @@ def _get_analytics_service(self): return build('analytics', 'v3', credentials=credentials) - def start_bundle(self): - pass - @staticmethod def _assert_all_list_names_are_present(any_execution): destination = any_execution.destination.destination_metadata @@ -60,7 +58,7 @@ def _assert_all_list_names_are_present(any_execution): raise ValueError('Missing destination information. Received {}'.format(str(destination))) @utils.safe_process( - logger=logging.getLogger('megalista.GoogleAnalyticsDataImportUploader')) + logger=logging.get_logger(LOGGER_NAME)) def process(self, batch: Batch, **kwargs): execution = batch.execution self._assert_all_list_names_are_present(execution) @@ -86,16 +84,16 @@ def process(self, batch: Batch, **kwargs): return [batch] except Exception as e: error_message = f'Error while delete GA Data Import files: {e}' - logging.getLogger("megalista.GoogleAnalyticsDataImportUploader").error(error_message) + logging.get_logger(LOGGER_NAME).error(error_message, execution=execution) self._add_error(execution, error_message) else: error_message = f"{data_import_name} - data import not found, please configure it in Google Analytics" - logging.getLogger("megalista.GoogleAnalyticsDataImportUploader").error(error_message) + logging.get_logger(LOGGER_NAME).error(error_message, execution=execution) self._add_error(execution, error_message) @staticmethod def _call_delete_api(analytics, data_import_name, ga_account_id, data_source_id, web_property_id): - logging.getLogger("megalista.GoogleAnalyticsDataImportUploader").info( + logging.get_logger(LOGGER_NAME).info( "Listing files from %s - %s" % (data_import_name, data_source_id)) uploads = analytics.management().uploads().list( @@ -106,14 +104,14 @@ def _call_delete_api(analytics, data_import_name, ga_account_id, data_source_id, file_ids = [upload.get('id') for upload in uploads.get('items', [])] if len(file_ids) == 0: - logging.getLogger("megalista.GoogleAnalyticsDataImportUploader").error( + logging.get_logger(LOGGER_NAME).error( "Data Source %s had no files to delete" % data_import_name) else: - logging.getLogger("megalista.GoogleAnalyticsDataImportUploader").info( + logging.get_logger(LOGGER_NAME).info( "File Ids: %s" % file_ids) - logging.getLogger("megalista.GoogleAnalyticsDataImportUploader").info( + logging.get_logger(LOGGER_NAME).info( "Deleting %s files from %s - %s" % (len(file_ids), data_import_name, data_source_id)) analytics.management().uploads().deleteUploadData( accountId=ga_account_id, diff --git a/megalista_dataflow/uploaders/google_analytics/google_analytics_data_import_eraser_test.py b/megalista_dataflow/uploaders/google_analytics/google_analytics_data_import_eraser_test.py index 84daf2f1..576fadbf 100644 --- a/megalista_dataflow/uploaders/google_analytics/google_analytics_data_import_eraser_test.py +++ b/megalista_dataflow/uploaders/google_analytics/google_analytics_data_import_eraser_test.py @@ -145,6 +145,6 @@ def test_files_deleted_with_success(mocker, eraser): _, kwargs = delete_call_mock.call_args # Check if really sent values from custom field - ids = kwargs['body'] + _ = kwargs['body'] # assert diff --git a/megalista_dataflow/uploaders/google_analytics/google_analytics_data_import_uploader.py b/megalista_dataflow/uploaders/google_analytics/google_analytics_data_import_uploader.py index ea32fbcd..e969e19a 100644 --- a/megalista_dataflow/uploaders/google_analytics/google_analytics_data_import_uploader.py +++ b/megalista_dataflow/uploaders/google_analytics/google_analytics_data_import_uploader.py @@ -13,7 +13,7 @@ # limitations under the License. # -import logging +from config import logging from typing import List, Dict from google.oauth2.credentials import Credentials @@ -25,6 +25,7 @@ from uploaders import utils from uploaders.uploaders import MegalistaUploader +LOGGER_NAME = 'megalista.GoogleAnalyticsDataImportUploader' class GoogleAnalyticsDataImportUploaderDoFn(MegalistaUploader): """ @@ -54,9 +55,6 @@ def _get_analytics_service(self): service = build('analytics', 'v3', credentials=credentials) return service - def start_bundle(self): - pass - @staticmethod def _assert_all_list_names_are_present(any_execution): destination = any_execution.destination.destination_metadata @@ -69,7 +67,7 @@ def _assert_all_list_names_are_present(any_execution): str(destination))) @utils.safe_process( - logger=logging.getLogger('megalista.GoogleAnalyticsDataImportUploader')) + logger=logging.get_logger(LOGGER_NAME)) def process(self, batch: Batch, **kwargs): execution = batch.execution self._assert_all_list_names_are_present(execution) @@ -106,12 +104,17 @@ def _do_upload_data(self, execution, web_property_id, data_import_name, ga_accou data_source_id, rows, web_property_id) except Exception as e: error_message = f'Error while uploading GA Data: {e}' - logging.getLogger('megalista.GoogleAnalyticsDataImportUploader').error(error_message) + logging.get_logger(LOGGER_NAME).error(error_message, execution=execution) self._add_error(execution, error_message) + execution.failed_records = execution.failed_records + len(rows) + else: + execution.successful_records = execution.successful_records + len(rows) else: error_message = f'{data_import_name} - data import not found, please configure it in Google Analytics' - logging.getLogger('megalista.GoogleAnalyticsDataImportUploader').error(error_message) + logging.get_logger(LOGGER_NAME).error(error_message, execution=execution) self._add_error(execution, error_message) + execution.failed_records = execution.failed_records + len(rows) + @staticmethod def prepare_csv(rows): @@ -138,7 +141,7 @@ def prepare_csv(rows): def _call_upload_api(self, analytics, data_import_name, ga_account_id, data_source_id, rows, web_property_id): - logging.getLogger('megalista.GoogleAnalyticsDataImportUploader').info( + logging.get_logger(LOGGER_NAME).info( 'Adding data to %s - %s' % (data_import_name, data_source_id)) csv = self.prepare_csv(rows) diff --git a/megalista_dataflow/uploaders/google_analytics/google_analytics_measurement_protocol.py b/megalista_dataflow/uploaders/google_analytics/google_analytics_measurement_protocol.py index d4320413..fc1615c0 100644 --- a/megalista_dataflow/uploaders/google_analytics/google_analytics_measurement_protocol.py +++ b/megalista_dataflow/uploaders/google_analytics/google_analytics_measurement_protocol.py @@ -13,7 +13,7 @@ # limitations under the License. -import logging +from config import logging import re from typing import Dict, Any from urllib.parse import quote @@ -32,8 +32,7 @@ def __init__(self, error_handler: ErrorHandler): self.API_URL = "https://www.google-analytics.com/batch" self.UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" - def start_bundle(self): - pass + def _format_hit(self, payload: Dict[str, Any]) -> str: return "&".join([key + "=" + quote(str(value)) for key, value in payload.items() if value is not None]) @@ -67,7 +66,7 @@ def build_hit(self, batch: Batch, row: Dict[str, Any]) -> Dict[str, Any]: # enhanced ecommerce parameters for events for key in row.keys(): - if (re.match('p([a]|[r]\d*[a-z]+)', key) # product details + if (re.match('p(a|r\d*[a-z]+)', key) # product details or re.match('t([irast])', key) # transaction details or re.match('cu', key) # currency code ): @@ -90,12 +89,12 @@ def build_hit(self, batch: Batch, row: Dict[str, Any]) -> Dict[str, Any]: payload["cu"] = row.get('currency_code') # Currency code. else: error_message = f"Hit type {hit_type} is not supported." - logging.getLogger("megalista.GoogleAnalyticsMeasurementProtocolUploader").error(error_message) + logging.get_logger("megalista.GoogleAnalyticsMeasurementProtocolUploader").error(error_message, execution=batch.execution) self._add_error(batch.execution, error_message) return payload - @utils.safe_process(logger=logging.getLogger("megalista.GoogleAnalyticsMeasurementProtocolUploader")) + @utils.safe_process(logger=logging.get_logger("megalista.GoogleAnalyticsMeasurementProtocolUploader")) def process(self, batch: Batch, **kwargs): rows = batch.elements @@ -108,7 +107,9 @@ def process(self, batch: Batch, **kwargs): response = requests.post(url=self.API_URL, data=payload) if response.status_code != 200: error_message = f"Error uploading to Analytics HTTP {response.status_code}: {response.raw}" - logging.getLogger("megalista.GoogleAnalyticsMeasurementProtocolUploader").error(error_message) + logging.get_logger("megalista.GoogleAnalyticsMeasurementProtocolUploader").error(error_message, execution=batch.execution) self._add_error(batch.execution, error_message) + batch.execution.failed_records = batch.execution.failed_records + len(rows) else: + batch.execution.successful_records = batch.execution.successful_records + len(rows) return [batch] diff --git a/megalista_dataflow/uploaders/google_analytics/google_analytics_user_list_uploader.py b/megalista_dataflow/uploaders/google_analytics/google_analytics_user_list_uploader.py index d5cd52a1..0ec9ddee 100644 --- a/megalista_dataflow/uploaders/google_analytics/google_analytics_user_list_uploader.py +++ b/megalista_dataflow/uploaders/google_analytics/google_analytics_user_list_uploader.py @@ -13,7 +13,7 @@ # limitations under the License. -import logging +from config import logging from google.oauth2.credentials import Credentials from googleapiclient.discovery import build @@ -50,7 +50,7 @@ def _create_list_if_doesnt_exist(self, analytics, web_property_id, view_ids, lis results = list( filter(lambda x: x['name'] == list_name, lists)) if len(results) == 0: - logging.getLogger().info('%s list does not exist, creating...' % list_name) + logging.get_logger().info('%s list does not exist, creating...' % list_name) response = analytics.management().remarketingAudience().insert( accountId=ga_account_id, @@ -64,15 +64,12 @@ def _create_list_if_doesnt_exist(self, analytics, web_property_id, view_ids, lis }], **list_definition }).execute() - id = response['id'] - logging.getLogger().info('%s created with id: %s' % (list_name, id)) + _id = response['id'] + logging.get_logger().info('%s created with id: %s' % (list_name, id)) else: - id = results[0]['id'] - logging.getLogger().info('%s found with id: %s' % (list_name, id)) - return id - - def start_bundle(self): - pass + _id = results[0]['id'] + logging.get_logger().info('%s found with id: %s' % (list_name, id)) + return _id def _create_list(self, web_property_id, view_id, user_id_list_name, buyer_custom_dim, ga_account_id, ads_customer_id, @@ -104,7 +101,7 @@ def _assert_all_list_names_are_present(any_execution): or not destination[5]: raise ValueError('Missing destination information. Received {}'.format(str(destination))) - @utils.safe_process(logger=logging.getLogger("megalista.GoogleAnalyticsUserListUploader")) + @utils.safe_process(logger=logging.get_logger("megalista.GoogleAnalyticsUserListUploader")) def process(self, batch: Batch, **kwargs): execution = batch.execution self._assert_all_list_names_are_present(execution) @@ -147,9 +144,9 @@ def _do_upload_data(self, execution, web_property_id, view_id, data_import_name, if len(results) == 1: - id = results[0]['id'] + _id = results[0]['id'] - logging.getLogger().info("Adding data to %s - %s" % (data_import_name, id)) + logging.get_logger().info("Adding data to %s - %s" % (data_import_name, _id)) body = '\n'.join([ '%s,%s' % (user_id_custom_dim, buyer_custom_dim), *['%s,%s' % (row['user_id'], row[custom_dim_field] if custom_dim_field else 'buyer') for row in rows] @@ -162,13 +159,19 @@ def _do_upload_data(self, execution, web_property_id, view_id, data_import_name, analytics.management().uploads().uploadData( accountId=ga_account_id, webPropertyId=web_property_id, - customDataSourceId=id, + customDataSourceId=_id, media_body=media).execute() except Exception as e: error_message = f'Error while uploading GA Data: {e}' - logging.getLogger().error(error_message) + logging.get_logger().error(error_message, execution=execution) self._add_error(execution, error_message) + execution.failed_records = execution.failed_records + len(rows) + else: + execution.successful_records = execution.successful_records + len(rows) + else: error_message = f"{data_import_name} - data import not found, please configure it in Google Analytics" - logging.getLogger().error(error_message) + logging.get_logger().error(error_message, execution=execution) self._add_error(execution, error_message) + execution.failed_records = execution.failed_records + len(rows) + diff --git a/megalista_dataflow/uploaders/support/transactional_events_results_writer.py b/megalista_dataflow/uploaders/support/transactional_events_results_writer.py index 02fc3aa1..9a3472a6 100644 --- a/megalista_dataflow/uploaders/support/transactional_events_results_writer.py +++ b/megalista_dataflow/uploaders/support/transactional_events_results_writer.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging from datetime import datetime import apache_beam as beam @@ -38,7 +38,7 @@ def __init__(self, dataflow_options: DataflowOptions, transactional_type: Transa self._dataflow_options = dataflow_options self._transactional_type = transactional_type - @utils.safe_process(logger=logging.getLogger("megalista.TransactionalEventsResultsWriter")) + @utils.safe_process(logger=logging.get_logger("megalista.TransactionalEventsResultsWriter")) def process(self, batch: Batch, *args, **kwargs): self._do_process(batch) return [batch.execution] diff --git a/megalista_dataflow/uploaders/uploaders.py b/megalista_dataflow/uploaders/uploaders.py index 5788fd64..b5858e84 100644 --- a/megalista_dataflow/uploaders/uploaders.py +++ b/megalista_dataflow/uploaders/uploaders.py @@ -31,5 +31,9 @@ def __init__(self, error_handler: ErrorHandler): def _add_error(self, execution: Execution, error_message: str): self._error_handler.add_error(execution, error_message) + def start_bundle(self): + # Nothing is needed when starting bundle. + pass + def finish_bundle(self): self._error_handler.notify_errors() diff --git a/megalista_dataflow/uploaders/utils.py b/megalista_dataflow/uploaders/utils.py index 34c35d6c..89622df8 100644 --- a/megalista_dataflow/uploaders/utils.py +++ b/megalista_dataflow/uploaders/utils.py @@ -13,13 +13,15 @@ # limitations under the License. import datetime -import logging +from config import logging import pytz import math from typing import Optional from models.execution import Batch from uploaders.uploaders import MegalistaUploader +from unittest.mock import ANY, MagicMock +from google.ads.googleads.errors import GoogleAdsException MAX_RETRIES = 3 @@ -71,14 +73,15 @@ def inner(*args, **kwargs): if not batch: logger.warning('Skipping upload, received no elements.') return - logger.info(f'Uploading {len(batch.elements)} rows...') + logger.info(f'Uploading {len(batch.elements)} rows...', execution=batch.execution) try: return func(*args, **kwargs) - except BaseException as e: + except Exception as e: self_._add_error(batch.execution, f'Error uploading data: {e}') - logger.error(f'Error uploading data for :{batch.elements}') - logger.error(e, exc_info=True) - logger.exception('Error uploading data.') + logger.warning(f'Error uploading data for :{batch.elements}', execution=batch.execution) + logger.warning(f'Exception type: {type(e).__name__}', execution=batch.execution) + logger.warning(e, exc_info=True, execution=batch.execution) + # logger.exception('Error uploading data.', execution=batch.execution) return inner @@ -93,7 +96,7 @@ def safe_call_api(function, logger, *args, **kwargs): def _do_safe_call_api(function, logger, current_retry, *args, **kwargs): try: return function(*args, *kwargs) - except Exception as e: + except Exception: if current_retry < MAX_RETRIES: logger.exception( f'Fail number {current_retry}. Stack track follows. Trying again.') @@ -119,7 +122,7 @@ def print_partial_error_messages(logger_name, action, response) -> Optional[str] partial_failure = getattr(response, 'partial_failure_error', None) if partial_failure is not None and partial_failure.message != '': error_message = f'Error on {action}: {partial_failure.message}.' - logging.getLogger(logger_name).error(error_message) + logging.get_logger(logger_name).info(error_message) results = getattr(response, 'results', []) for result in results: gclid = getattr(result, 'gclid', None) @@ -131,6 +134,17 @@ def print_partial_error_messages(logger_name, action, response) -> Optional[str] else: message = f'item {result} uploaded.' - logging.getLogger(logger_name).debug(message) + logging.get_logger(logger_name).debug(message) return error_message + +def update_execution_counters(execution, elements, response): + failed_records = 0 + partial_failure = getattr(response, 'partial_failure_error', None) + if partial_failure is not None: + details = getattr(partial_failure, 'details', []) + failed_records = len(details) + successful_records = len(elements) - failed_records + execution.successful_records = execution.successful_records + successful_records + execution.failed_records = execution.failed_records + failed_records + \ No newline at end of file diff --git a/megalista_dataflow/uploaders/utils_test.py b/megalista_dataflow/uploaders/utils_test.py index cf935e85..508b6b1e 100644 --- a/megalista_dataflow/uploaders/utils_test.py +++ b/megalista_dataflow/uploaders/utils_test.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging +from config import logging from error.error_handling import ErrorHandler from error.error_handling_test import MockErrorNotifier @@ -26,7 +26,7 @@ class MockUploader(MegalistaUploader): def __init__(self, error_handler: ErrorHandler): super().__init__(error_handler) - @utils.safe_process(logger=logging.getLogger('megalista.UtilsTest')) + @utils.safe_process(logger=logging.get_logger('megalista.UtilsTest')) def process(self, batch: Batch, **kwargs): self._add_error(batch.execution, error_message)