[circle-resizer-dredd-recipe-test] Add circle-resizer dredd tests

compiler/circle-resizer-dredd-recipe-test/CMakeLists.txt

@@ -0,0 +1,117 @@
+if(NOT ENABLE_TEST)
+  return()
+endif(NOT ENABLE_TEST)
+
+nnas_include(TargetRequire)
+
+unset(REQUIRED_TARGETS)
+list(APPEND REQUIRED_TARGETS circle-inspect)
+list(APPEND REQUIRED_TARGETS circle-verify)
+list(APPEND REQUIRED_TARGETS circle_resizer)
+list(APPEND REQUIRED_TARGETS dredd_rule_lib)
+TargetRequire_Return(${REQUIRED_TARGETS})
+
+unset(TEST_DEPS)
+unset(TEST_NAMES)
+
+get_target_property(ARTIFACTS_BIN_PATH testDataGenerator BINARY_DIR)
+
+set(oneValueArgs NEW_INPUTS_SIZES)
+set(multiValueArgs "")
+
+macro(Add RECIPE)
+  cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+  set(NEW_INPUTS_SIZES "")
+  if(ARG_NEW_INPUTS_SIZES)
+    set(NEW_INPUTS_SIZES "--input_shapes" "${ARG_NEW_INPUTS_SIZES}")
+  endif()
+
+  set(CIRCLE_PATH "${ARTIFACTS_BIN_PATH}/${RECIPE}.circle")
+  set(RESIZED_CIRCLE_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RECIPE}.resized.circle")
+
+  # Generate resized .circle
+  add_custom_command(OUTPUT ${RESIZED_CIRCLE_PATH}
+    COMMAND $<TARGET_FILE:circle_resizer> --input_path ${CIRCLE_PATH} --output_path ${RESIZED_CIRCLE_PATH} ${NEW_INPUTS_SIZES}
+    DEPENDS 
+      circle_resizer
+      ${CIRCLE_PATH}
+    COMMENT "Generate ${RECIPE}.resized.circle"
+  )
+
+  list(APPEND TEST_DEPS ${RESIZED_CIRCLE_PATH})
+  list(APPEND TEST_NAMES ${RECIPE})
+endmacro(Add)
+
+
+# Read "test.lst"
+include("test.lst")
+
+##
+## Copy testall
+##
+set(TEST_RUNNER "${CMAKE_CURRENT_BINARY_DIR}/testall.sh")
+set(TEST_RUNNER_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/testall.sh")
+
+add_custom_command(
+  OUTPUT ${TEST_RUNNER}
+  COMMAND ${CMAKE_COMMAND} -E copy "${TEST_RUNNER_SOURCE}" "${TEST_RUNNER}"
+  DEPENDS ${TEST_RUNNER_SOURCE}
+  COMMENT "Generate test runner"
+)
+
+list(APPEND TEST_DEPS "${TEST_RUNNER}")
+
+###
+### Generate test.config
+###
+set(TEST_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/test.config")
+
+add_custom_command(
+  OUTPUT ${TEST_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E remove -f ${TEST_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'CIRCLE_INSPECT_PATH=\"$<TARGET_FILE:circle-inspect>\"' >> ${TEST_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'CIRCLE_VERIFY_PATH=\"$<TARGET_FILE:circle-verify>\"' >> ${TEST_CONFIG}
+  COMMAND ${CMAKE_COMMAND} -E echo 'CIRCLE_RESIZER_PATH=\"$<TARGET_FILE:circle_resizer>\"' >> ${TEST_CONFIG}
+  DEPENDS
+    circle-inspect
+    circle-verify
+    circle_resizer
+  COMMENT "Generate test configuration"
+)
+
+list(APPEND TEST_DEPS "${TEST_CONFIG}")
+
+#
+# copy rule-lib.sh (a library of shell script functions)
+#
+
+# getting path for rule-lib.sh in dredd-rule-lib
+get_target_property(DREDD_RULE_LIB_DIR dredd_rule_lib BINARY_DIR)
+
+set(RULE_LIB_SOURCE_PATH "${DREDD_RULE_LIB_DIR}/rule-lib.sh")
+set(RULE_LIB_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/rule-lib.sh")
+
+add_custom_command(
+  OUTPUT ${RULE_LIB_BINARY_PATH}
+  COMMAND ${CMAKE_COMMAND} -E copy "${RULE_LIB_SOURCE_PATH}" "${RULE_LIB_BINARY_PATH}"
+  DEPENDS ${RULE_LIB_SOURCE_PATH}
+  COMMENT "Generate rule lib"
+)
+
+list(APPEND TEST_DEPS "${RULE_LIB_BINARY_PATH}")
+
+# Generate dependencies
+add_custom_target(circle-resizer_dredd_recipe_test ALL DEPENDS ${TEST_DEPS})
+add_dependencies(circle-resizer_dredd_recipe_test common_artifacts_deps)
+
+get_target_property(ARTIFACTS_BIN_PATH testDataGenerator BINARY_DIR)
+
+# Run tests
+add_test(
+  NAME resizer_dredd_recipe_test
+  COMMAND "${TEST_RUNNER}"
+          "${TEST_CONFIG}"
+          "${ARTIFACTS_BIN_PATH}"
+          ${TEST_NAMES}
+)

compiler/circle-resizer-dredd-recipe-test/README.md

@@ -0,0 +1,31 @@
+# circle-resizer-dredd-recipe-test
+
+It tests non-functional conditions of a circle model resized by circle-resizer.
+
+## How to add a test?
+
+1. Create a directory under `res/TensorFlowLiteRecipes/` or `res/CircleRecipes/`.
+
+2. Make a recipe (`test.recipe`) for a model under the directory.
+
+3. Make a rule (`test.rule`) you want to test under the directory. Note, that you can find more information about dredd-test-rules in _dredd-rule-lib_ module.
+
+4. Add a test to `test.lst` in this module using `Add` macro.
+   ```
+   Add(RECIPE_DIR NEW_INPUTS_SIZES)
+   ```
+   - `NEW_INPUTS_SIZES`: New shapes of Circle model inputs in comma-separated format like `[1,2,3],[4,5]` for a model with 2 inputs.
+
+## Example
+
+```
+# TensorFlowLiteRecipes
+res/TensorFlowLiteRecipes/PRelu_000
+├── test.recipe     # What you want to test
+└── test.rule       # Non-functional conditions to be satisfied
+
+# test.lst
+...
+Add(PRelu_000 NEW_INPUTS_SIZES [1,4,4,5],[1,1,5])
+...
+```

compiler/circle-resizer-dredd-recipe-test/requires.cmake

@@ -0,0 +1,5 @@
+require("circle-inspect")
+require("circle-resizer")
+require("circle-verify")
+require("common-artifacts")
+require("dredd-rule-lib")

compiler/circle-resizer-dredd-recipe-test/test.lst

@@ -0,0 +1,19 @@
+## EXAMPLE
+#
+# Add(RECIPE_DIR NEW_INPUTS_SIZES [1,2,3])
+#
+
+## TFLITE RECIPE
+
+# two inputs, one output
+Add(PRelu_002 NEW_INPUTS_SIZES [1,4,4,5],[1,1,5])
+# scalar output
+Add(ReduceAny_dynamic_004 NEW_INPUTS_SIZES [4,5,6])
+# change rank
+Add(GreaterEqual_001 NEW_INPUTS_SIZES [1,2,3],[1,2,3])
+# one inputs, two outputs
+Add(Split_001 NEW_INPUTS_SIZES [8,1,2])
+# bigger graph
+Add(Net_FullyConnected_Gelu_001 NEW_INPUTS_SIZES [2,16])
+# from dynamic to static
+Add(Inf_StridedSlice_002 NEW_INPUTS_SIZES [1,10,10,5])

compiler/circle-resizer-dredd-recipe-test/testall.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+# Need at least 2 arguments
+if [[ $# -lt 2 ]]; then
+  echo "USAGE: $0 ..."
+  echo
+  echo "ARGUMENTS:"
+  echo "  [test.config path]"
+  echo "  [WORKDIR]"
+  echo "  [Prefix1]"
+  echo "  [Prefix2]"
+  echo "  ..."
+  exit 255
+fi
+
+WORKDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+CONFIG_PATH="$1"; shift
+RESOURCE_DIR="$1"; shift
+
+source "${CONFIG_PATH}"
+
+echo "-- Found circle-resizer: ${CIRCLE_RESIZER_PATH}"
+echo "-- Found circle-inspect: ${CIRCLE_INSPECT_PATH}"
+echo "-- Found circle-verify: ${CIRCLE_VERIFY_PATH}"
+echo "-- Found common-artifacts: ${RESOURCE_DIR}"
+
+TESTED=()
+PASSED=()
+FAILED=()
+
+pushd ${WORKDIR}
+while [[ $# -ne 0 ]]; do
+  PREFIX="$1"; shift
+
+  TESTED+=("${PREFIX}")
+
+  PASSED_TAG="${PREFIX}.passed"
+
+  rm -f "${PASSED_TAG}"
+
+  cat > "${PREFIX}.log" <(
+    exec 2>&1
+
+    echo "-- Found circle: ${PREFIX}.resized.circle"
+
+    # Exit immediately if any command fails
+    set -e
+    # Show commands
+    set -x
+
+    #
+    # Check if rule is satisfied
+    #
+
+    # Note: turn off 'command printing'. Otherwise printing will be so messy
+    set +x
+
+    # (COMPILED_FILE, INSPECT_PROG_PATH, VERIFY_PROG_PATH, ERROR_LOG) must be set for rule-lib.sh
+    COMPILED_FILE="${PREFIX}.resized.circle"
+    INSPECT_PROG_PATH=${CIRCLE_INSPECT_PATH}
+    VERIFY_PROG_PATH=${CIRCLE_VERIFY_PATH}
+    ERROR_LOG="${PREFIX}.error"
+
+    rm -f "${ERROR_LOG}"
+
+    # in case error while running rule-lib.sh, prints error msg
+    trap 'echo "** ERROR **" ; cat "${ERROR_LOG}"' ERR
+
+    source rule-lib.sh
+    source "${RESOURCE_DIR}/${PREFIX}.rule"
+
+    # unset
+    trap - ERR
+    set -x
+
+    # At this point, the exit code of all commands is 0
+    # If not 0, execution of this script ends because of "set -e"
+    touch "${PASSED_TAG}"
+  )
+
+  if [[ -f "${PASSED_TAG}" ]]; then
+    PASSED+=("$PREFIX")
+  else
+    FAILED+=("$PREFIX")
+  fi
+done
+popd
+
+if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then
+  echo "FAILED"
+  for TEST in "${FAILED[@]}"
+  do
+    echo "- ${TEST}"
+  done
+  exit 255
+fi
+
+echo "PASSED"
+exit 0