From 21c51b885472a1a2ef6668fe8fbc3c3051ce6b97 Mon Sep 17 00:00:00 2001
From: Quentin Kaiser <quentin.kaiser@onekey.com>
Date: Mon, 6 Feb 2023 19:40:52 +0100
Subject: [PATCH 1/2] chore(deps): replace ubi-reader fork by official PyPi
 version.

We cannot use git hosted dependencies if we want to publish unblob to
PyPi. We therefore moved from our fork of ubi-reader to the official
package of ubi-reader.

Our fork was synced with upstream except for one UI change we did to get
rid of superfluous directories created by ubireader_extract_images.
Since the maintainer of ubi-reader does not want to include it upstream,
we moved that logic into the UBIExtractor class.

See https://github.com/jrspruitt/ubi_reader/pull/67 for more details.
---
 poetry.lock                       | 22 +++++++---------------
 pyproject.toml                    |  2 +-
 unblob/handlers/filesystem/ubi.py | 16 +++++++++++++++-
 3 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index a9aef5878d..7f0637299f 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -820,22 +820,12 @@ optional = false
 python-versions = ">=3.7"
 
 [[package]]
-name = "ubi_reader"
-version = "0.8.0"
-description = ""
+name = "ubi-reader"
+version = "0.8.5"
+description = "Extract files from UBI and UBIFS images."
 category = "main"
 optional = false
 python-versions = "*"
-develop = false
-
-[package.dependencies]
-python-lzo = "*"
-
-[package.source]
-type = "git"
-url = "https://github.com/onekey-sec/ubi_reader.git"
-reference = "8c956d47b28af4085366e2acfee8d3ba016f6e90"
-resolved_reference = "8c956d47b28af4085366e2acfee8d3ba016f6e90"
 
 [[package]]
 name = "urllib3"
@@ -916,7 +906,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8"
-content-hash = "d8f0619745d724d934e1cef6442ccb588d5901dc18b512187a28c787b5198bd1"
+content-hash = "c6151a08e3dd2dd2191daaf14b631ce4ff50bd5e87903b6ff8887db26fb5aadb"
 
 [metadata.files]
 arpy = [
@@ -1492,7 +1482,9 @@ typing-extensions = [
     {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"},
     {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"},
 ]
-ubi_reader = []
+ubi-reader = [
+    {file = "ubi_reader-0.8.5.tar.gz", hash = "sha256:0d1467e582a1c9ba574bb1f941e45fb74f92ab8acfcba1586a8a464b0432e81e"},
+]
 urllib3 = [
     {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"},
     {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"},
diff --git a/pyproject.toml b/pyproject.toml
index 610d7a03cb..d2e47306c9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -18,7 +18,7 @@ attrs = "^21.2.0"
 structlog = "^21.2.0"
 arpy = "^2.2.0"
 rarfile = "^4.0"
-ubi-reader = { git = "https://github.com/onekey-sec/ubi_reader.git", rev = "8c956d47b28af4085366e2acfee8d3ba016f6e90" }
+ubi-reader = "^0.8.5"
 python-lzo = "^1.14"
 cstruct = "2.1"
 jefferson = { git = "https://github.com/onekey-sec/jefferson.git", rev = "51de1f5ba3255ed707bb402bee89e442e11b55c1" }
diff --git a/unblob/handlers/filesystem/ubi.py b/unblob/handlers/filesystem/ubi.py
index 3c9df02c26..8224f972c0 100644
--- a/unblob/handlers/filesystem/ubi.py
+++ b/unblob/handlers/filesystem/ubi.py
@@ -1,4 +1,6 @@
+import shutil
 import statistics
+from pathlib import Path
 from typing import Optional
 
 from structlog import get_logger
@@ -95,6 +97,18 @@ def calculate_chunk(self, file: File, start_offset: int) -> Optional[ValidChunk]
         )
 
 
+class UBIExtractor(Command):
+    def extract(self, inpath: Path, outdir: Path):
+        super().extract(inpath, outdir)
+        # ubireader_extract_images creates a superfluous directory named
+        # after the UBI file (inpath here), so we simply move the files up
+        # and delete the remaining directory.
+        superfluous_dir_path = outdir.joinpath(inpath.name)
+        for file_path in superfluous_dir_path.iterdir():
+            shutil.move(file_path.as_posix(), outdir.as_posix())
+        shutil.rmtree(superfluous_dir_path.as_posix())
+
+
 class UBIHandler(Handler):
     NAME = "ubi"
 
@@ -102,7 +116,7 @@ class UBIHandler(Handler):
 
     PATTERNS = [HexString("55 42 49 23 01  // UBI# and version 1")]
 
-    EXTRACTOR = Command("ubireader_extract_images", "{inpath}", "-o", "{outdir}")
+    EXTRACTOR = UBIExtractor("ubireader_extract_images", "{inpath}", "-o", "{outdir}")
 
     def _guess_peb_size(self, file: File) -> int:
         # Since we don't know the PEB size, we need to guess it. At the moment we just find the

From 3ead9da00db984528290a1343ed6e74dd1960215 Mon Sep 17 00:00:00 2001
From: Quentin Kaiser <quentin.kaiser@onekey.com>
Date: Mon, 6 Feb 2023 20:30:38 +0100
Subject: [PATCH 2/2] chore(ci): explicitly include dev dependencies when
 installing dependencies with poetry.

---
 .github/actions/setup-dependencies/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/setup-dependencies/action.yml b/.github/actions/setup-dependencies/action.yml
index b8cd098d44..cc50a5f2cb 100644
--- a/.github/actions/setup-dependencies/action.yml
+++ b/.github/actions/setup-dependencies/action.yml
@@ -48,5 +48,5 @@ runs:
         key: venv-${{ hashFiles('poetry.lock') }}-${{ inputs.python-version }}
 
     - name: Poetry install
-      run: UNBLOB_BUILD_RUST_EXTENSION=1 poetry install
+      run: UNBLOB_BUILD_RUST_EXTENSION=1 poetry install --with dev
       shell: bash