From a83f449d8a4246095eb64c67d7c4af04d275c618 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Mon, 6 Apr 2026 15:44:40 -0400
Subject: [PATCH 01/34] return 400 for invalid patch request

---
 sdx_controller/controllers/l2vpn_controller.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index 50b5d734..113c5393 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -206,6 +206,7 @@ def place_connection(body):
 
     if code // 100 != 2:
         conn_status = ConnectionStateMachine.State.REJECTED
+        body, _ = connection_state_machine(body, conn_status)
         db_instance.update_field_in_json(
             MongoCollections.CONNECTIONS,
             service_id,
@@ -259,6 +260,19 @@ def patch_connection(service_id, body=None):  # noqa: E501
 
     logger.info(f"Gathered connexion JSON: {new_body}")
 
+    # Validate the new request body before making any change to the existing connection.
+    # This is to avoid the case where we have already removed the original connection but the new request body is invalid, which will cause the connection to be deleted but not re-created.
+    # We can reuse the same validation function used in place_connection since the request body for patch_connection has the same schema as place_connection.
+    #
+    te_manager = current_app.te_manager  # Assuming te_manager is accessible like this
+    try:
+        # Validate the new request body
+        te_manager.generate_traffic_matrix(connection_request=new_body)
+    except Exception as request_err:
+        logger.error("ERROR: invalid patch request: " + str(request_err))
+        return f"Error: patch request is no valid: {request_err}", 400
+
+    logger.info("Modifying connection")
     # Get roll back connection before removing connection
     rollback_conn_body = copy.deepcopy(body)
     body.update(new_body)

From 0761eca172c4a2443b0a91e4dce608a8e62d731c Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Mon, 6 Apr 2026 16:20:05 -0400
Subject: [PATCH 02/34] pce dependency

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 8a3a4187..2fe2192d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,7 +29,7 @@ dependencies = [
     "pika >= 1.2.0",
     "dataset",
     "pymongo > 3.0",
-    "sdx-pce @ git+https://github.com/atlanticwave-sdx/pce@v3.2.1",
+    "sdx-pce @ git+https://github.com/atlanticwave-sdx/pce@302-port-nni-removed",
 ]
 
 [project.optional-dependencies]

From bdeb054c8f983bf2d45e89c8ae705cb118929003 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Mon, 6 Apr 2026 16:33:38 -0400
Subject: [PATCH 03/34] sm transition in place_connection

---
 sdx_controller/controllers/l2vpn_controller.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index 113c5393..b1ee2070 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -205,7 +205,7 @@ def place_connection(body):
     reason, code = connection_handler.place_connection(current_app.te_manager, body)
 
     if code // 100 != 2:
-        conn_status = ConnectionStateMachine.State.REJECTED
+        conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
         body, _ = connection_state_machine(body, conn_status)
         db_instance.update_field_in_json(
             MongoCollections.CONNECTIONS,
@@ -213,6 +213,9 @@ def place_connection(body):
             "status",
             str(conn_status),
         )
+    else:
+        conn_status = ConnectionStateMachine.State.REJECTED
+        body, _ = connection_state_machine(body, conn_status)
     logger.info(
         f"place_connection result: ID: {service_id} reason='{reason}', code={code}"
     )

From 74eef97ccc297bcb8734a9ca09eabf0522c32fd3 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Mon, 6 Apr 2026 16:39:02 -0400
Subject: [PATCH 04/34] complete previous commit

---
 sdx_controller/controllers/l2vpn_controller.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index b1ee2070..fa84a2bf 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -194,9 +194,6 @@ def place_connection(body):
     # used in lc_message_handler to count the oxp success response
     body["oxp_success_count"] = 0
 
-    conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
-    body, _ = connection_state_machine(body, conn_status)
-
     db_instance.add_key_value_pair_to_db(MongoCollections.CONNECTIONS, service_id, body)
 
     logger.info(
@@ -204,7 +201,7 @@ def place_connection(body):
     )
     reason, code = connection_handler.place_connection(current_app.te_manager, body)
 
-    if code // 100 != 2:
+    if code // 100 == 2:
         conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
         body, _ = connection_state_machine(body, conn_status)
         db_instance.update_field_in_json(

From 343d7397be1dfe7b59151bebbeba88eb2fca7bbe Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Mon, 6 Apr 2026 17:21:50 -0400
Subject: [PATCH 05/34] adding id to the patch request

---
 sdx_controller/controllers/l2vpn_controller.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index fa84a2bf..b4eeeee5 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -260,6 +260,9 @@ def patch_connection(service_id, body=None):  # noqa: E501
 
     logger.info(f"Gathered connexion JSON: {new_body}")
 
+    if "id" not in new_body:
+        new_body["id"] = service_id
+
     # Validate the new request body before making any change to the existing connection.
     # This is to avoid the case where we have already removed the original connection but the new request body is invalid, which will cause the connection to be deleted but not re-created.
     # We can reuse the same validation function used in place_connection since the request body for patch_connection has the same schema as place_connection.

From 2cd537da6d6b6566d8180551f7d99ee3b7791abf Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Mon, 6 Apr 2026 17:50:29 -0400
Subject: [PATCH 06/34] pce v3.2.1

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 2fe2192d..8a3a4187 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,7 +29,7 @@ dependencies = [
     "pika >= 1.2.0",
     "dataset",
     "pymongo > 3.0",
-    "sdx-pce @ git+https://github.com/atlanticwave-sdx/pce@302-port-nni-removed",
+    "sdx-pce @ git+https://github.com/atlanticwave-sdx/pce@v3.2.1",
 ]
 
 [project.optional-dependencies]

From b15f4eb19583d326dc14298ad1f7381730135453 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Mon, 6 Apr 2026 18:47:00 -0400
Subject: [PATCH 07/34] save rejected

---
 sdx_controller/controllers/l2vpn_controller.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index b4eeeee5..bf307f3a 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -213,6 +213,12 @@ def place_connection(body):
     else:
         conn_status = ConnectionStateMachine.State.REJECTED
         body, _ = connection_state_machine(body, conn_status)
+        db_instance.update_field_in_json(
+            MongoCollections.CONNECTIONS,
+            service_id,
+            "status",
+            str(conn_status),
+        )
     logger.info(
         f"place_connection result: ID: {service_id} reason='{reason}', code={code}"
     )

From c655a1ae464ba4d8ca3aa3ec74aed6d75150845a Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 00:03:35 -0400
Subject: [PATCH 08/34] patching SM

---
 .../controllers/l2vpn_controller.py           | 72 ++++++++++++++-----
 1 file changed, 56 insertions(+), 16 deletions(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index bf307f3a..0c28d423 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -194,7 +194,7 @@ def place_connection(body):
     # used in lc_message_handler to count the oxp success response
     body["oxp_success_count"] = 0
 
-    db_instance.add_key_value_pair_to_db(MongoCollections.CONNECTIONS, service_id, body)
+    # db_instance.add_key_value_pair_to_db(MongoCollections.CONNECTIONS, service_id, body)
 
     logger.info(
         f"Handling request {service_id} with te_manager: {current_app.te_manager}"
@@ -286,11 +286,18 @@ def patch_connection(service_id, body=None):  # noqa: E501
     rollback_conn_body = copy.deepcopy(body)
     body.update(new_body)
 
-    body, _ = connection_state_machine(body, ConnectionStateMachine.State.MODIFYING)
+    conn_status = ConnectionStateMachine.State.MODIFYING
+    body, _ = connection_state_machine(body, conn_status)
+    db_instance.update_field_in_json(
+        MongoCollections.CONNECTIONS,
+        service_id,
+        "status",
+        str(conn_status),
+    )
 
     body["oxp_success_count"] = 0
 
-    db_instance.add_key_value_pair_to_db(MongoCollections.CONNECTIONS, service_id, body)
+    # db_instance.add_key_value_pair_to_db(MongoCollections.CONNECTIONS, service_id, body)
 
     try:
         logger.info("Removing connection")
@@ -299,9 +306,13 @@ def patch_connection(service_id, body=None):  # noqa: E501
         )
 
         if remove_conn_code // 100 != 2:
-            body, _ = connection_state_machine(body, ConnectionStateMachine.State.DOWN)
-            db_instance.add_key_value_pair_to_db(
-                MongoCollections.CONNECTIONS, service_id, body
+            conn_status = ConnectionStateMachine.State.DOWN
+            body, _ = connection_state_machine(body, conn_status)
+            db_instance.update_field_in_json(
+                MongoCollections.CONNECTIONS,
+                service_id,
+                "status",
+                str(conn_status),
             )
             response = {
                 "service_id": service_id,
@@ -313,20 +324,32 @@ def patch_connection(service_id, body=None):  # noqa: E501
         logger.info(f"Removed connection: {service_id}")
     except Exception as e:
         logger.info(f"Delete failed (connection id: {service_id}): {e}")
+        conn_status = ConnectionStateMachine.State.DOWN
+        body, _ = connection_state_machine(body, conn_status)
+        db_instance.update_field_in_json(
+            MongoCollections.CONNECTIONS,
+            service_id,
+            "status",
+            str(conn_status),
+        )
         return f"Failed, reason: {e}", 500
 
     logger.info(
-        f"Placing new connection {service_id} with te_manager: {current_app.te_manager}"
+        f"Modifying: Placing new connection {service_id} with te_manager: {current_app.te_manager}"
     )
 
-    body, _ = connection_state_machine(
-        body, ConnectionStateMachine.State.UNDER_PROVISIONING
-    )
-    db_instance.add_key_value_pair_to_db(MongoCollections.CONNECTIONS, service_id, body)
     reason, code = connection_handler.place_connection(current_app.te_manager, body)
 
     if code // 100 == 2:
         # Service created successfully
+        conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
+        body, _ = connection_state_machine(body, conn_status)
+        db_instance.update_field_in_json(
+            MongoCollections.CONNECTIONS,
+            service_id,
+            "status",
+            str(conn_status),
+        )
         code = 201
         logger.info(f"Placed: ID: {service_id} reason='{reason}', code={code}")
         response = {
@@ -335,8 +358,15 @@ def patch_connection(service_id, body=None):  # noqa: E501
             "reason": reason,
         }
         return response, code
-    else:
-        body, _ = connection_state_machine(body, ConnectionStateMachine.State.DOWN)
+
+    conn_status = ConnectionStateMachine.State.DOWN
+    body, _ = connection_state_machine(body, conn_status)
+    db_instance.update_field_in_json(
+        MongoCollections.CONNECTIONS,
+        service_id,
+        "status",
+        str(conn_status),
+    )
 
     logger.info(
         f"Failed to place new connection. ID: {service_id} reason='{reason}', code={code}"
@@ -348,8 +378,6 @@ def patch_connection(service_id, body=None):  # noqa: E501
     rollback_conn_body["status"] = str(ConnectionStateMachine.State.REQUESTED)
     # used in lc_message_handler to count the oxp success response
     rollback_conn_body["oxp_success_count"] = 0
-    conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
-    rollback_conn_body, _ = connection_state_machine(rollback_conn_body, conn_status)
 
     conn_request = rollback_conn_body
     conn_request["id"] = service_id
@@ -358,8 +386,20 @@ def patch_connection(service_id, body=None):  # noqa: E501
         rollback_conn_reason, rollback_conn_code = connection_handler.place_connection(
             current_app.te_manager, conn_request
         )
-        if rollback_conn_code // 100 != 2:
+        if rollback_conn_code // 100 == 2:
+            conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
+            rollback_conn_body, _ = connection_state_machine(
+                rollback_conn_body, conn_status
+            )
+            db_instance.update_field_in_json(
+                MongoCollections.CONNECTIONS,
+                service_id,
+                "status",
+                str(conn_status),
+            )
+        else:
             conn_status = ConnectionStateMachine.State.REJECTED
+            body, _ = connection_state_machine(body, conn_status)
             db_instance.update_field_in_json(
                 MongoCollections.CONNECTIONS,
                 service_id,

From 0c217076d44defab66f6ccf2011eef0d154f7490 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 00:23:52 -0400
Subject: [PATCH 09/34] edit

---
 sdx_controller/controllers/l2vpn_controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index 0c28d423..afc7c53f 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -194,7 +194,7 @@ def place_connection(body):
     # used in lc_message_handler to count the oxp success response
     body["oxp_success_count"] = 0
 
-    # db_instance.add_key_value_pair_to_db(MongoCollections.CONNECTIONS, service_id, body)
+    db_instance.add_key_value_pair_to_db(MongoCollections.CONNECTIONS, service_id, body)
 
     logger.info(
         f"Handling request {service_id} with te_manager: {current_app.te_manager}"

From ac7bb768d8739a60ca6b3d104d30fedc3063b526 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 01:58:06 -0400
Subject: [PATCH 10/34] patching db logic

---
 .../controllers/l2vpn_controller.py           | 23 +++++--------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index afc7c53f..a6023385 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -295,10 +295,6 @@ def patch_connection(service_id, body=None):  # noqa: E501
         str(conn_status),
     )
 
-    body["oxp_success_count"] = 0
-
-    # db_instance.add_key_value_pair_to_db(MongoCollections.CONNECTIONS, service_id, body)
-
     try:
         logger.info("Removing connection")
         remove_conn_reason, remove_conn_code = connection_handler.remove_connection(
@@ -337,18 +333,16 @@ def patch_connection(service_id, body=None):  # noqa: E501
     logger.info(
         f"Modifying: Placing new connection {service_id} with te_manager: {current_app.te_manager}"
     )
-
+    # reset: the original one has been removed from db
     reason, code = connection_handler.place_connection(current_app.te_manager, body)
 
+    body["oxp_success_count"] = 0
     if code // 100 == 2:
         # Service created successfully
         conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
         body, _ = connection_state_machine(body, conn_status)
-        db_instance.update_field_in_json(
-            MongoCollections.CONNECTIONS,
-            service_id,
-            "status",
-            str(conn_status),
+        db_instance.add_key_value_pair_to_db(
+            MongoCollections.CONNECTIONS, service_id, body
         )
         code = 201
         logger.info(f"Placed: ID: {service_id} reason='{reason}', code={code}")
@@ -361,15 +355,10 @@ def patch_connection(service_id, body=None):  # noqa: E501
 
     conn_status = ConnectionStateMachine.State.DOWN
     body, _ = connection_state_machine(body, conn_status)
-    db_instance.update_field_in_json(
-        MongoCollections.CONNECTIONS,
-        service_id,
-        "status",
-        str(conn_status),
-    )
+    db_instance.add_key_value_pair_to_db(MongoCollections.CONNECTIONS, service_id, body)
 
     logger.info(
-        f"Failed to place new connection. ID: {service_id} reason='{reason}', code={code}"
+        f"Modifying: Failed to place new connection. ID: {service_id} reason='{reason}', code={code}"
     )
     logger.info("Rolling back to old connection.")
 

From 7478d00bdd7c58a7aa155b03d6f54a646d633f34 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 09:22:55 -0400
Subject: [PATCH 11/34] patching return code 400 if rollback succeeds after
 patch failed

---
 sdx_controller/controllers/l2vpn_controller.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index a6023385..a15030e1 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -386,6 +386,8 @@ def patch_connection(service_id, body=None):  # noqa: E501
                 "status",
                 str(conn_status),
             )
+            # still return 400 to indicate the patch request is not successful, since we have already rolled back to original connection, which is under provisioning state, so the connection is not down and not failed.
+            rollback_conn_code = 400
         else:
             conn_status = ConnectionStateMachine.State.REJECTED
             body, _ = connection_state_machine(body, conn_status)

From 3c2d66b3ff10125db7125c93d4443980ed4a3761 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 11:01:53 -0400
Subject: [PATCH 12/34] update connection status, not add connection, in db
 when failure

---
 sdx_controller/handlers/connection_handler.py | 30 +++++++++++++--
 sdx_controller/handlers/lc_message_handler.py | 37 ++++++-------------
 2 files changed, 38 insertions(+), 29 deletions(-)

diff --git a/sdx_controller/handlers/connection_handler.py b/sdx_controller/handlers/connection_handler.py
index 6fc1e18e..742957e6 100644
--- a/sdx_controller/handlers/connection_handler.py
+++ b/sdx_controller/handlers/connection_handler.py
@@ -522,24 +522,46 @@ def handle_link_failure(self, te_manager, failed_links):
                     connection, _ = connection_state_machine(
                         connection, ConnectionStateMachine.State.RECOVERING
                     )
-                    connection["oxp_success_count"] = 0
+
                     self.db_instance.add_key_value_pair_to_db(
                         MongoCollections.CONNECTIONS, service_id, connection
                     )
                     _reason, code = self.place_connection(te_manager, connection)
-                    if code // 100 != 2:
+                    connection["oxp_success_count"] = 0
+                    if code // 100 == 2:
+                        # Service created successfully
+                        conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
+                        connection, _ = connection_state_machine(
+                            connection, conn_status
+                        )
+                        self.db_instance.update_field_in_json(
+                            MongoCollections.CONNECTIONS,
+                            service_id,
+                            "status",
+                            str(conn_status),
+                        )
+                        code = 201
+                    else:
                         connection, _ = connection_state_machine(
                             connection, ConnectionStateMachine.State.ERROR
                         )
-                        self.db_instance.add_key_value_pair_to_db(
+                        self.db_instance.update_field_in_json(
                             MongoCollections.CONNECTIONS,
                             service_id,
-                            connection,
+                            "status",
+                            str(conn_status),
                         )
+                        code = 400
 
                     logger.info(
                         f"place_connection result: ID: {service_id} reason='{_reason}', code={code}"
                     )
+                    response = {
+                        "service_id": service_id,
+                        "status": parse_conn_status(connection["status"]),
+                        "reason": _reason,
+                    }
+                    return response, code
 
     def handle_uni_ports_up_to_down(self, uni_ports_up_to_down):
         """
diff --git a/sdx_controller/handlers/lc_message_handler.py b/sdx_controller/handlers/lc_message_handler.py
index 76230361..17dc21e5 100644
--- a/sdx_controller/handlers/lc_message_handler.py
+++ b/sdx_controller/handlers/lc_message_handler.py
@@ -93,6 +93,7 @@ def process_lc_json_msg(
                 logger.info(f"Could not find breakdown for {service_id}")
                 return None
 
+            conn_status = connection.get("status")
             oxp_number = len(breakdown)
             oxp_success_count = connection.get("oxp_success_count", 0)
             lc_domain = msg_json.get("lc_domain")
@@ -108,46 +109,32 @@ def process_lc_json_msg(
                 if msg_json.get("operation") != "delete":
                     oxp_success_count += 1
                     connection["oxp_success_count"] = oxp_success_count
+                    conn_status = ConnectionStateMachine.State.UP
                     if oxp_success_count == oxp_number:
-                        if connection.get("status") and (
-                            connection.get("status")
-                            == str(ConnectionStateMachine.State.RECOVERING)
-                        ):
-                            connection, _ = connection_state_machine(
-                                connection,
-                                ConnectionStateMachine.State.UNDER_PROVISIONING,
-                            )
                         connection, _ = connection_state_machine(
-                            connection, ConnectionStateMachine.State.UP
+                            connection, conn_status
                         )
             else:
-                if connection.get("status") and (
-                    connection.get("status")
-                    == str(ConnectionStateMachine.State.RECOVERING)
-                ):
-                    connection, _ = connection_state_machine(
-                        connection, ConnectionStateMachine.State.ERROR
-                    )
-                elif (
+                if (
                     connection.get("status")
                     and connection.get("status")
-                    != str(ConnectionStateMachine.State.DOWN)
+                    == str(ConnectionStateMachine.State.MODIFYING)
                     and connection.get("status")
-                    != str(ConnectionStateMachine.State.ERROR)
+                    == str(ConnectionStateMachine.State.UNDER_PROVISIONING)
                 ):
-                    connection, _ = connection_state_machine(
-                        connection, ConnectionStateMachine.State.DOWN
-                    )
+                    conn_status = ConnectionStateMachine.State.DOWN
+                    connection, _ = connection_state_machine(connection, conn_status)
 
             # ToDo: eg: if 3 oxps in the breakdowns: (1) all up: up (2) parital down: remove_connection()
             # release successful oxp circuits if some are down: remove_connection() (3) count the responses
             # to finalize the status of the connection.
-            self.db_instance.add_key_value_pair_to_db(
+            self.db_instance.update_field_in_json(
                 MongoCollections.CONNECTIONS,
                 service_id,
-                connection,
+                "status",
+                str(conn_status),
             )
-            logger.info("Connection updated: " + service_id)
+            logger.info("Connection updated: " + str(connection))
             return
 
         # topology message RPC from OXP: no exchange name is defined.

From 4905b666ac32d77bb7c57d3407dd62b1928d7615 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 12:02:54 -0400
Subject: [PATCH 13/34] update oxp_response in db

---
 sdx_controller/handlers/lc_message_handler.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/sdx_controller/handlers/lc_message_handler.py b/sdx_controller/handlers/lc_message_handler.py
index 17dc21e5..93ae5e8b 100644
--- a/sdx_controller/handlers/lc_message_handler.py
+++ b/sdx_controller/handlers/lc_message_handler.py
@@ -134,6 +134,12 @@ def process_lc_json_msg(
                 "status",
                 str(conn_status),
             )
+            self.db_instance.update_field_in_json(
+                MongoCollections.CONNECTIONS,
+                service_id,
+                "oxp_response",
+                oxp_response,
+            )
             logger.info("Connection updated: " + str(connection))
             return
 

From b6598ad5fbf758e11efe81702738560379486a34 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 12:56:48 -0400
Subject: [PATCH 14/34] log oxp_success_count

---
 sdx_controller/handlers/lc_message_handler.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/sdx_controller/handlers/lc_message_handler.py b/sdx_controller/handlers/lc_message_handler.py
index 93ae5e8b..5f19f568 100644
--- a/sdx_controller/handlers/lc_message_handler.py
+++ b/sdx_controller/handlers/lc_message_handler.py
@@ -109,8 +109,11 @@ def process_lc_json_msg(
                 if msg_json.get("operation") != "delete":
                     oxp_success_count += 1
                     connection["oxp_success_count"] = oxp_success_count
-                    conn_status = ConnectionStateMachine.State.UP
+                    logger.info(
+                        f"Update oxp_success_count: {oxp_success_count}; oxp_number: {oxp_number}"
+                    )
                     if oxp_success_count == oxp_number:
+                        conn_status = ConnectionStateMachine.State.UP
                         connection, _ = connection_state_machine(
                             connection, conn_status
                         )
@@ -140,6 +143,12 @@ def process_lc_json_msg(
                 "oxp_response",
                 oxp_response,
             )
+            self.db_instance.update_field_in_json(
+                MongoCollections.CONNECTIONS,
+                service_id,
+                "oxp_success_count",
+                oxp_success_count,
+            )
             logger.info("Connection updated: " + str(connection))
             return
 

From d18a72289989376b4e75cc689a170b9c3b0a926a Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 13:29:54 -0400
Subject: [PATCH 15/34] reset oxp resonse in db when recovering

---
 sdx_controller/handlers/connection_handler.py | 26 +++++++++++++------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/sdx_controller/handlers/connection_handler.py b/sdx_controller/handlers/connection_handler.py
index 742957e6..cf287652 100644
--- a/sdx_controller/handlers/connection_handler.py
+++ b/sdx_controller/handlers/connection_handler.py
@@ -5,6 +5,7 @@
 import traceback
 from typing import Tuple
 
+from sdx_controller.models import connection
 from sdx_datamodel.connection_sm import ConnectionStateMachine
 from sdx_datamodel.constants import Constants, MessageQueueNames, MongoCollections
 from sdx_datamodel.parsing.exceptions import (
@@ -522,12 +523,13 @@ def handle_link_failure(self, te_manager, failed_links):
                     connection, _ = connection_state_machine(
                         connection, ConnectionStateMachine.State.RECOVERING
                     )
-
+                    connection["oxp_success_count"] = 0
+                    connection["oxp_response"] = {}
                     self.db_instance.add_key_value_pair_to_db(
                         MongoCollections.CONNECTIONS, service_id, connection
                     )
                     _reason, code = self.place_connection(te_manager, connection)
-                    connection["oxp_success_count"] = 0
+
                     if code // 100 == 2:
                         # Service created successfully
                         conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
@@ -594,9 +596,13 @@ def handle_uni_ports_up_to_down(self, uni_ports_up_to_down):
                         logger.debug(f"Cannot find connection {service_id} in DB.")
                         continue
                     logger.info(f"Updating connection {service_id} status to 'down'.")
-                    connection["status"] = "DOWN"
-                    self.db_instance.add_key_value_pair_to_db(
-                        MongoCollections.CONNECTIONS, service_id, connection
+                    conn_status = "DOWN"
+                    connection["status"] = conn_status
+                    self.db_instance.update_field_in_json(
+                        MongoCollections.CONNECTIONS,
+                        service_id,
+                        "status",
+                        str(conn_status),
                     )
                     logger.debug(f"Connection status updated for {service_id}")
             else:
@@ -635,9 +641,13 @@ def handle_uni_ports_down_to_up(self, uni_ports_down_to_up):
                         continue
 
                     logger.info(f"Updating connection {service_id} status to 'up'.")
-                    connection["status"] = "UP"
-                    self.db_instance.add_key_value_pair_to_db(
-                        MongoCollections.CONNECTIONS, service_id, connection
+                    conn_status = "UP"
+                    connection["status"] = conn_status
+                    self.db_instance.update_field_in_json(
+                        MongoCollections.CONNECTIONS,
+                        service_id,
+                        "status",
+                        str(conn_status),
                     )
                     logger.debug(f"Connection status updated for {service_id}")
 

From 2536b820b7c17448f6bcd074f4e40b65d81a335a Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 13:31:55 -0400
Subject: [PATCH 16/34] isort

---
 sdx_controller/handlers/connection_handler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdx_controller/handlers/connection_handler.py b/sdx_controller/handlers/connection_handler.py
index cf287652..14bcd0b3 100644
--- a/sdx_controller/handlers/connection_handler.py
+++ b/sdx_controller/handlers/connection_handler.py
@@ -5,7 +5,6 @@
 import traceback
 from typing import Tuple
 
-from sdx_controller.models import connection
 from sdx_datamodel.connection_sm import ConnectionStateMachine
 from sdx_datamodel.constants import Constants, MessageQueueNames, MongoCollections
 from sdx_datamodel.parsing.exceptions import (
@@ -21,6 +20,7 @@
 )
 
 from sdx_controller.messaging.topic_queue_producer import TopicQueueProducer
+from sdx_controller.models import connection
 from sdx_controller.models.simple_link import SimpleLink
 from sdx_controller.utils.parse_helper import ParseHelper
 

From 0b7eb9d55114ff3eb00c3eb7574ed9ad363a9e16 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 17:39:58 -0400
Subject: [PATCH 17/34] reset oxp_response attribute in db in recovering and
 patch

---
 sdx_controller/controllers/l2vpn_controller.py | 5 ++++-
 sdx_controller/handlers/connection_handler.py  | 3 ++-
 sdx_controller/handlers/lc_message_handler.py  | 5 ++---
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index a15030e1..06d7fa19 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -189,7 +189,8 @@ def place_connection(body):
         body["id"] = service_id
         logger.info(f"Request has no ID. Generated ID: {service_id}")
 
-    body["status"] = str(ConnectionStateMachine.State.REQUESTED)
+    conn_status = ConnectionStateMachine.State.REQUESTED
+    body["status"] = str(conn_status)
 
     # used in lc_message_handler to count the oxp success response
     body["oxp_success_count"] = 0
@@ -337,6 +338,7 @@ def patch_connection(service_id, body=None):  # noqa: E501
     reason, code = connection_handler.place_connection(current_app.te_manager, body)
 
     body["oxp_success_count"] = 0
+    body["oxp_response"] = {}
     if code // 100 == 2:
         # Service created successfully
         conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
@@ -367,6 +369,7 @@ def patch_connection(service_id, body=None):  # noqa: E501
     rollback_conn_body["status"] = str(ConnectionStateMachine.State.REQUESTED)
     # used in lc_message_handler to count the oxp success response
     rollback_conn_body["oxp_success_count"] = 0
+    rollback_conn_body["oxp_response"] = {}
 
     conn_request = rollback_conn_body
     conn_request["id"] = service_id
diff --git a/sdx_controller/handlers/connection_handler.py b/sdx_controller/handlers/connection_handler.py
index 14bcd0b3..6de35270 100644
--- a/sdx_controller/handlers/connection_handler.py
+++ b/sdx_controller/handlers/connection_handler.py
@@ -544,8 +544,9 @@ def handle_link_failure(self, te_manager, failed_links):
                         )
                         code = 201
                     else:
+                        conn_status = ConnectionStateMachine.State.ERROR
                         connection, _ = connection_state_machine(
-                            connection, ConnectionStateMachine.State.ERROR
+                            connection, conn_status
                         )
                         self.db_instance.update_field_in_json(
                             MongoCollections.CONNECTIONS,
diff --git a/sdx_controller/handlers/lc_message_handler.py b/sdx_controller/handlers/lc_message_handler.py
index 5f19f568..54821d71 100644
--- a/sdx_controller/handlers/lc_message_handler.py
+++ b/sdx_controller/handlers/lc_message_handler.py
@@ -118,11 +118,10 @@ def process_lc_json_msg(
                             connection, conn_status
                         )
             else:
-                if (
+                if connection.get("status") and (
                     connection.get("status")
-                    and connection.get("status")
                     == str(ConnectionStateMachine.State.MODIFYING)
-                    and connection.get("status")
+                    or connection.get("status")
                     == str(ConnectionStateMachine.State.UNDER_PROVISIONING)
                 ):
                     conn_status = ConnectionStateMachine.State.DOWN

From 759518f09ba8c2e9179424421ffdfb77fd70b904 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 18:52:11 -0400
Subject: [PATCH 18/34] try pce@302-port-nni-removed

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 8a3a4187..2fe2192d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,7 +29,7 @@ dependencies = [
     "pika >= 1.2.0",
     "dataset",
     "pymongo > 3.0",
-    "sdx-pce @ git+https://github.com/atlanticwave-sdx/pce@v3.2.1",
+    "sdx-pce @ git+https://github.com/atlanticwave-sdx/pce@302-port-nni-removed",
 ]
 
 [project.optional-dependencies]

From cad300d6a4bc8a8853f777b545869ed341658d24 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 19:32:39 -0400
Subject: [PATCH 19/34] log a debug

---
 sdx_controller/handlers/connection_handler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sdx_controller/handlers/connection_handler.py b/sdx_controller/handlers/connection_handler.py
index 6de35270..b7432e37 100644
--- a/sdx_controller/handlers/connection_handler.py
+++ b/sdx_controller/handlers/connection_handler.py
@@ -201,6 +201,9 @@ def _send_breakdown_to_lc(self, breakdown, operation, connection_request):
             }
 
             if operation == "delete":
+                logger.debug(
+                    f"Handling delete operation for connection {connection_request}"
+                )
                 oxp_response = connection_request.get("oxp_response")
 
                 # evc_id is the service_id in the OXP response, it differs from the service_id in the connection.

From 8af94337ed675374ae8aa4c4aa783fe5744f352f Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 20:50:50 -0400
Subject: [PATCH 20/34] conn_status before roll_back in patch

---
 sdx_controller/controllers/l2vpn_controller.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index 06d7fa19..46a3d03a 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -357,7 +357,6 @@ def patch_connection(service_id, body=None):  # noqa: E501
 
     conn_status = ConnectionStateMachine.State.DOWN
     body, _ = connection_state_machine(body, conn_status)
-    db_instance.add_key_value_pair_to_db(MongoCollections.CONNECTIONS, service_id, body)
 
     logger.info(
         f"Modifying: Failed to place new connection. ID: {service_id} reason='{reason}', code={code}"
@@ -373,6 +372,9 @@ def patch_connection(service_id, body=None):  # noqa: E501
 
     conn_request = rollback_conn_body
     conn_request["id"] = service_id
+    db_instance.add_key_value_pair_to_db(
+        MongoCollections.CONNECTIONS, service_id, conn_request
+    )
 
     try:
         rollback_conn_reason, rollback_conn_code = connection_handler.place_connection(

From 7be30831ec855552dc347354ef961276c5ee95ea Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 21:18:28 -0400
Subject: [PATCH 21/34] catch mal-formated oxp_response with L2VPN Deleted

---
 sdx_controller/handlers/connection_handler.py | 20 +++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/sdx_controller/handlers/connection_handler.py b/sdx_controller/handlers/connection_handler.py
index b7432e37..bdd24741 100644
--- a/sdx_controller/handlers/connection_handler.py
+++ b/sdx_controller/handlers/connection_handler.py
@@ -204,14 +204,18 @@ def _send_breakdown_to_lc(self, breakdown, operation, connection_request):
                 logger.debug(
                     f"Handling delete operation for connection {connection_request}"
                 )
-                oxp_response = connection_request.get("oxp_response")
-
-                # evc_id is the service_id in the OXP response, it differs from the service_id in the connection.
-                evc_id = (
-                    oxp_response.get(domain_name, [None, {}])[1].get("service_id")
-                    if oxp_response
-                    else None
-                )
+                try:
+                    oxp_response = connection_request.get("oxp_response")
+                    # evc_id is the service_id in the OXP response, it differs from the service_id in the connection.
+                    evc_id = (
+                        oxp_response.get(domain_name, [None, {}])[1].get("service_id")
+                        if oxp_response
+                        else None
+                    )
+                except Exception as e:
+                    logger.error(
+                        f"Error occurred while processing OXP response in delete: {e}"
+                    )
 
                 if not oxp_response or not evc_id:
                     return (

From 95636f70e8518d0a3e31010f07d7b70bb01a7ff3 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Tue, 7 Apr 2026 22:03:07 -0400
Subject: [PATCH 22/34] catch mal-formated oxp_response

---
 sdx_controller/handlers/connection_handler.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sdx_controller/handlers/connection_handler.py b/sdx_controller/handlers/connection_handler.py
index bdd24741..49fd0810 100644
--- a/sdx_controller/handlers/connection_handler.py
+++ b/sdx_controller/handlers/connection_handler.py
@@ -204,6 +204,8 @@ def _send_breakdown_to_lc(self, breakdown, operation, connection_request):
                 logger.debug(
                     f"Handling delete operation for connection {connection_request}"
                 )
+                oxp_response = None
+                evc_id = None
                 try:
                     oxp_response = connection_request.get("oxp_response")
                     # evc_id is the service_id in the OXP response, it differs from the service_id in the connection.

From 1936fc28191dffcd4bb42f0c730f1582f6f3b103 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Sat, 11 Apr 2026 19:26:48 -0400
Subject: [PATCH 23/34] finer return code in patch

---
 sdx_controller/controllers/l2vpn_controller.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index 46a3d03a..e266225e 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -280,7 +280,8 @@ def patch_connection(service_id, body=None):  # noqa: E501
         te_manager.generate_traffic_matrix(connection_request=new_body)
     except Exception as request_err:
         logger.error("ERROR: invalid patch request: " + str(request_err))
-        return f"Error: patch request is no valid: {request_err}", 400
+        error_code = str(request_err).split("Code: ")[-1].replace(")", "").strip()
+        return f"Error: patch request is no valid: {request_err}", int(error_code)
 
     logger.info("Modifying connection")
     # Get roll back connection before removing connection
@@ -392,7 +393,7 @@ def patch_connection(service_id, body=None):  # noqa: E501
                 str(conn_status),
             )
             # still return 400 to indicate the patch request is not successful, since we have already rolled back to original connection, which is under provisioning state, so the connection is not down and not failed.
-            rollback_conn_code = 400
+            rollback_conn_code = code
         else:
             conn_status = ConnectionStateMachine.State.REJECTED
             body, _ = connection_state_machine(body, conn_status)

From ab48a841f98333b62160dafe8ed86a30f5658545 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Sat, 11 Apr 2026 22:39:49 -0400
Subject: [PATCH 24/34] add some sleep time after remove_connection call

---
 sdx_controller/controllers/l2vpn_controller.py | 3 ++-
 sdx_controller/handlers/connection_handler.py  | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index e266225e..3fa88a33 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -1,6 +1,7 @@
 import copy
 import logging
 import os
+import time
 import uuid
 
 import connexion
@@ -331,7 +332,7 @@ def patch_connection(service_id, body=None):  # noqa: E501
             str(conn_status),
         )
         return f"Failed, reason: {e}", 500
-
+    time.sleep(10)
     logger.info(
         f"Modifying: Placing new connection {service_id} with te_manager: {current_app.te_manager}"
     )
diff --git a/sdx_controller/handlers/connection_handler.py b/sdx_controller/handlers/connection_handler.py
index 49fd0810..2aca7eee 100644
--- a/sdx_controller/handlers/connection_handler.py
+++ b/sdx_controller/handlers/connection_handler.py
@@ -529,6 +529,8 @@ def handle_link_failure(self, te_manager, failed_links):
 
                     logger.debug("Removed connection:")
                     logger.debug(connection)
+                    time.sleep(10)
+
                     connection, _ = connection_state_machine(
                         connection, ConnectionStateMachine.State.RECOVERING
                     )

From 5c38c53d2281cfc51afde748a4b380b49d0cc01c Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Wed, 15 Apr 2026 17:05:32 -0400
Subject: [PATCH 25/34] streamlined the SM and db operation to avoid
 inconsistence with lc_message_handler thread, added a missed db saving in the
 middle of patch

---
 .../controllers/l2vpn_controller.py           | 82 +++++++++++++------
 sdx_controller/handlers/connection_handler.py | 48 +++++++++--
 2 files changed, 94 insertions(+), 36 deletions(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index 3fa88a33..bc4e23a8 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -92,7 +92,15 @@ def delete_connection(service_id):
 
         logger.info(f"Removing connection: {service_id} {connection.get('status')}")
 
-        connection_handler.remove_connection(current_app.te_manager, service_id, "API")
+        remove_reason, remove_code = connection_handler.remove_connection(
+            current_app.te_manager, service_id, "API"
+        )
+        if remove_code // 100 != 2:
+            logger.info(
+                f"Delete failed (connection id: {service_id}): "
+                f"reason='{remove_reason}', code={remove_code}"
+            )
+            return remove_reason, remove_code
         db_instance.mark_deleted(MongoCollections.CONNECTIONS, f"{service_id}")
         db_instance.mark_deleted(MongoCollections.BREAKDOWNS, f"{service_id}")
     except Exception as e:
@@ -204,14 +212,15 @@ def place_connection(body):
     reason, code = connection_handler.place_connection(current_app.te_manager, body)
 
     if code // 100 == 2:
-        conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
-        body, _ = connection_state_machine(body, conn_status)
-        db_instance.update_field_in_json(
-            MongoCollections.CONNECTIONS,
-            service_id,
-            "status",
-            str(conn_status),
-        )
+        # conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
+        # body, _ = connection_state_machine(body, conn_status)
+        # db_instance.update_field_in_json(
+        #    MongoCollections.CONNECTIONS,
+        #    service_id,
+        #    "status",
+        #    str(conn_status),
+        # )
+        logger.info(f"place_connection succeeds: ID: {service_id} body='{body}'")
     else:
         conn_status = ConnectionStateMachine.State.REJECTED
         body, _ = connection_state_machine(body, conn_status)
@@ -281,8 +290,20 @@ def patch_connection(service_id, body=None):  # noqa: E501
         te_manager.generate_traffic_matrix(connection_request=new_body)
     except Exception as request_err:
         logger.error("ERROR: invalid patch request: " + str(request_err))
-        error_code = str(request_err).split("Code: ")[-1].replace(")", "").strip()
-        return f"Error: patch request is no valid: {request_err}", int(error_code)
+        error_code = getattr(request_err, "request_code", None)
+        if not isinstance(error_code, int):
+            # Backward-compatible fallback for exception strings like "... (Code: 400)".
+            error_code = 400
+            err_text = str(request_err)
+            if "Code:" in err_text:
+                candidate = err_text.split("Code:")[-1].replace(")", "").strip()
+                try:
+                    error_code = int(candidate)
+                except (TypeError, ValueError):
+                    logger.warning(
+                        f"Could not parse error code from patch validation error: {err_text}"
+                    )
+        return f"Error: patch request is not valid: {request_err}", error_code
 
     logger.info("Modifying connection")
     # Get roll back connection before removing connection
@@ -336,18 +357,22 @@ def patch_connection(service_id, body=None):  # noqa: E501
     logger.info(
         f"Modifying: Placing new connection {service_id} with te_manager: {current_app.te_manager}"
     )
-    # reset: the original one has been removed from db
+    # Reset: remove_connection archives/deletes the original entry,
+    # so persist the patched request before re-placement.
+    conn_status = ConnectionStateMachine.State.REQUESTED
+    body["status"] = str(conn_status)
+    db_instance.add_key_value_pair_to_db(MongoCollections.CONNECTIONS, service_id, body)
     reason, code = connection_handler.place_connection(current_app.te_manager, body)
 
     body["oxp_success_count"] = 0
     body["oxp_response"] = {}
     if code // 100 == 2:
         # Service created successfully
-        conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
-        body, _ = connection_state_machine(body, conn_status)
-        db_instance.add_key_value_pair_to_db(
-            MongoCollections.CONNECTIONS, service_id, body
-        )
+        # conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
+        # body, _ = connection_state_machine(body, conn_status)
+        # db_instance.add_key_value_pair_to_db(
+        #    MongoCollections.CONNECTIONS, service_id, body
+        # )
         code = 201
         logger.info(f"Placed: ID: {service_id} reason='{reason}', code={code}")
         response = {
@@ -378,21 +403,22 @@ def patch_connection(service_id, body=None):  # noqa: E501
         MongoCollections.CONNECTIONS, service_id, conn_request
     )
 
+    rollback_conn_reason = "Rollback attempt did not complete"
     try:
         rollback_conn_reason, rollback_conn_code = connection_handler.place_connection(
             current_app.te_manager, conn_request
         )
         if rollback_conn_code // 100 == 2:
-            conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
-            rollback_conn_body, _ = connection_state_machine(
-                rollback_conn_body, conn_status
-            )
-            db_instance.update_field_in_json(
-                MongoCollections.CONNECTIONS,
-                service_id,
-                "status",
-                str(conn_status),
-            )
+            # conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
+            # rollback_conn_body, _ = connection_state_machine(
+            #    rollback_conn_body, conn_status
+            # )
+            # db_instance.update_field_in_json(
+            #    MongoCollections.CONNECTIONS,
+            #    service_id,
+            #    "status",
+            #    str(conn_status),
+            # )
             # still return 400 to indicate the patch request is not successful, since we have already rolled back to original connection, which is under provisioning state, so the connection is not down and not failed.
             rollback_conn_code = code
         else:
@@ -404,6 +430,7 @@ def patch_connection(service_id, body=None):  # noqa: E501
                 "status",
                 str(conn_status),
             )
+            rollback_conn_code = 500
         logger.info(
             f"Roll back connection result: ID: {service_id} reason='{rollback_conn_reason}', code={rollback_conn_code}"
         )
@@ -416,6 +443,7 @@ def patch_connection(service_id, body=None):  # noqa: E501
             str(conn_status),
         )
         logger.info(f"Rollback failed (connection id: {service_id}): {e}")
+        rollback_conn_reason = f"Rollback failed: {e}"
         rollback_conn_code = 500
 
     response = {
diff --git a/sdx_controller/handlers/connection_handler.py b/sdx_controller/handlers/connection_handler.py
index 2aca7eee..e0e47410 100644
--- a/sdx_controller/handlers/connection_handler.py
+++ b/sdx_controller/handlers/connection_handler.py
@@ -296,6 +296,17 @@ def place_connection(
                     MongoCollections.BREAKDOWNS, connection_request["id"], breakdown
                 )
                 self._process_port(connection_request["id"], ctx.ingress_port, "post")
+                conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
+                connection_request, _ = connection_state_machine(
+                    connection_request, conn_status
+                )
+                self.db_instance.update_field_in_json(
+                    MongoCollections.CONNECTIONS,
+                    connection_request["id"],
+                    "status",
+                    str(conn_status),
+                )
+
                 status, code = self._send_breakdown_to_lc(
                     breakdown, "post", connection_request
                 )
@@ -353,6 +364,16 @@ def place_connection(
                 operation="post",
                 connection_request=connection_request,
             )
+            conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
+            connection_request, _ = connection_state_machine(
+                connection_request, conn_status
+            )
+            self.db_instance.update_field_in_json(
+                MongoCollections.CONNECTIONS,
+                connection_request["id"],
+                "status",
+                str(conn_status),
+            )
             status, code = self._send_breakdown_to_lc(
                 breakdown, "post", connection_request
             )
@@ -436,6 +457,12 @@ def remove_connection(
             status, code = self._send_breakdown_to_lc(
                 breakdown, "delete", connection_request
             )
+            if code // 100 != 2:
+                logger.error(
+                    f"Could not publish delete breakdown for {service_id}: "
+                    f"reason='{status}', code={code}"
+                )
+                return status, code
             self._process_path_to_db(
                 te_manager, operation="delete", connection_request=connection_request
             )
@@ -543,15 +570,18 @@ def handle_link_failure(self, te_manager, failed_links):
 
                     if code // 100 == 2:
                         # Service created successfully
-                        conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
-                        connection, _ = connection_state_machine(
-                            connection, conn_status
-                        )
-                        self.db_instance.update_field_in_json(
-                            MongoCollections.CONNECTIONS,
-                            service_id,
-                            "status",
-                            str(conn_status),
+                        # conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING
+                        # connection, _ = connection_state_machine(
+                        #    connection, conn_status
+                        # )
+                        # self.db_instance.update_field_in_json(
+                        #    MongoCollections.CONNECTIONS,
+                        #    service_id,
+                        #    "status",
+                        #    str(conn_status),
+                        # )
+                        logger.info(
+                            f"link failure rerouting: place_connection succeeds: ID: {service_id} connection='{connection}'"
                         )
                         code = 201
                     else:

From 6290dc67c5ff4fed8ff25b51ef5a2c680940c3b0 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Wed, 15 Apr 2026 17:22:49 -0400
Subject: [PATCH 26/34] lint copy

---
 sdx_controller/controllers/l2vpn_controller.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index 28101376..9eeee4c0 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -1,3 +1,4 @@
+import copy
 import logging
 import os
 import time

From 8abde6b99451f5dc1f309fc67deb726de27057b5 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Wed, 15 Apr 2026 17:50:38 -0400
Subject: [PATCH 27/34] minor

---
 .../controllers/l2vpn_controller.py           | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index 9eeee4c0..b4c9b4a2 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -321,8 +321,6 @@ def patch_connection(service_id, body=None):  # noqa: E501
 
     try:
         logger.info("Removing connection")
-        # Get roll back connection before removing connection
-        rollback_conn_body = body
         remove_conn_reason, remove_conn_code = connection_handler.remove_connection(
             current_app.te_manager, service_id, "API"
         )
@@ -392,14 +390,6 @@ def patch_connection(service_id, body=None):  # noqa: E501
     )
     logger.info("Rolling back to old connection.")
 
-    if not rollback_conn_body:
-        response = {
-            "service_id": service_id,
-            "status": parse_conn_status(body["status"]),
-            "reason": f"Failure, unable to rollback to last successful L2VPN: {reason}",
-        }
-        return response, code
-
     # because above placement failed, so re-place the original connection request.
 
     rollback_conn_body["status"] = str(ConnectionStateMachine.State.REQUESTED)
@@ -445,6 +435,13 @@ def patch_connection(service_id, body=None):  # noqa: E501
             f"Roll back connection result: ID: {service_id} reason='{rollback_conn_reason}', code={rollback_conn_code}"
         )
     except Exception as e:
+        conn_status = ConnectionStateMachine.State.REJECTED
+        db_instance.update_field_in_json(
+            MongoCollections.CONNECTIONS,
+            service_id,
+            "status",
+            str(conn_status),
+        )
         logger.info(f"Rollback failed (connection id: {service_id}): {e}")
         rollback_conn_reason = f"Rollback failed: {e}"
         rollback_conn_code = 500
@@ -454,7 +451,7 @@ def patch_connection(service_id, body=None):  # noqa: E501
         "reason": f"Failure, rolled back to last successful L2VPN: {reason}",
         "status": parse_conn_status(conn_request["status"]),
     }
-    return response, code
+    return response, rollback_conn_code
 
 
 def get_archived_connections_by_id(service_id):

From 0301fc8ec294bed763e97845dd2b28ba280ea833 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Wed, 15 Apr 2026 19:59:35 -0400
Subject: [PATCH 28/34] avoid oxp_response being none

---
 sdx_controller/controllers/l2vpn_controller.py |  7 ++++++-
 sdx_controller/handlers/connection_handler.py  | 13 ++++++-------
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index b4c9b4a2..d7ec124c 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -446,10 +446,15 @@ def patch_connection(service_id, body=None):  # noqa: E501
         rollback_conn_reason = f"Rollback failed: {e}"
         rollback_conn_code = 500
 
+    current_conn = db_instance.get_value_from_db(
+        MongoCollections.CONNECTIONS, f"{service_id}"
+    )
     response = {
         "service_id": service_id,
         "reason": f"Failure, rolled back to last successful L2VPN: {reason}",
-        "status": parse_conn_status(conn_request["status"]),
+        "status": parse_conn_status(
+            current_conn.get("status", "") if current_conn else ""
+        ),
     }
     return response, rollback_conn_code
 
diff --git a/sdx_controller/handlers/connection_handler.py b/sdx_controller/handlers/connection_handler.py
index e0e47410..c8a2bc78 100644
--- a/sdx_controller/handlers/connection_handler.py
+++ b/sdx_controller/handlers/connection_handler.py
@@ -214,18 +214,17 @@ def _send_breakdown_to_lc(self, breakdown, operation, connection_request):
                         if oxp_response
                         else None
                     )
+                    if not oxp_response or not evc_id:
+                        return (
+                            "Connection does not have OXP response, cannot remove connection",
+                            404,
+                        )
+                    mq_link["evc_id"] = evc_id
                 except Exception as e:
                     logger.error(
                         f"Error occurred while processing OXP response in delete: {e}"
                     )
 
-                if not oxp_response or not evc_id:
-                    return (
-                        "Connection does not have OXP response, cannot remove connection",
-                        404,
-                    )
-                mq_link["evc_id"] = evc_id
-
             producer = TopicQueueProducer(
                 timeout=5, exchange_name=exchange_name, routing_key=domain_name
             )

From 7d46722cf1a86de113a1c0b7e272fca79ca9dd5e Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Wed, 15 Apr 2026 20:06:30 -0400
Subject: [PATCH 29/34] donot return remove_connection() output because of
 unittest where no oxp_response is present

---
 sdx_controller/controllers/l2vpn_controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index d7ec124c..fce64364 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -100,7 +100,7 @@ def delete_connection(service_id):
                 f"Delete failed (connection id: {service_id}): "
                 f"reason='{remove_reason}', code={remove_code}"
             )
-            return remove_reason, remove_code
+            # return remove_reason, remove_code
         db_instance.mark_deleted(MongoCollections.CONNECTIONS, f"{service_id}")
         db_instance.mark_deleted(MongoCollections.BREAKDOWNS, f"{service_id}")
     except Exception as e:

From 39763fb81b553f4301c81778869a1401b2fc72b7 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Wed, 15 Apr 2026 21:09:22 -0400
Subject: [PATCH 30/34] local variable issue

---
 sdx_controller/controllers/l2vpn_controller.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index fce64364..4b8109a0 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -234,9 +234,16 @@ def place_connection(body):
         f"place_connection result: ID: {service_id} reason='{reason}', code={code}"
     )
 
+    current_conn = db_instance.get_value_from_db(
+        MongoCollections.CONNECTIONS, f"{service_id}"
+    )
     response = {
         "service_id": service_id,
-        "status": parse_conn_status(str(conn_status)),
+        "status": parse_conn_status(
+            current_conn.get("status", str(conn_status))
+            if current_conn
+            else str(conn_status)
+        ),
         "reason": reason,
     }
 

From 001d3de317c027825045bd0419133669e8a69a89 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Wed, 15 Apr 2026 21:52:39 -0400
Subject: [PATCH 31/34] status in patch

---
 sdx_controller/controllers/l2vpn_controller.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index 4b8109a0..f3f6dfe5 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -368,11 +368,11 @@ def patch_connection(service_id, body=None):  # noqa: E501
     # so persist the patched request before re-placement.
     conn_status = ConnectionStateMachine.State.REQUESTED
     body["status"] = str(conn_status)
+    body["oxp_success_count"] = 0
+    body["oxp_response"] = {}
     db_instance.add_key_value_pair_to_db(MongoCollections.CONNECTIONS, service_id, body)
     reason, code = connection_handler.place_connection(current_app.te_manager, body)
 
-    body["oxp_success_count"] = 0
-    body["oxp_response"] = {}
     if code // 100 == 2:
         # Service created successfully
         # conn_status = ConnectionStateMachine.State.UNDER_PROVISIONING

From 036df22420f78cf2e0001f9834eee5ca761a9313 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Thu, 16 Apr 2026 10:29:14 -0400
Subject: [PATCH 32/34] comment out a wrong return in link failure handling

---
 sdx_controller/controllers/l2vpn_controller.py |  3 ---
 sdx_controller/handlers/connection_handler.py  | 17 ++++++++++-------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/sdx_controller/controllers/l2vpn_controller.py b/sdx_controller/controllers/l2vpn_controller.py
index f3f6dfe5..5f0a020c 100644
--- a/sdx_controller/controllers/l2vpn_controller.py
+++ b/sdx_controller/controllers/l2vpn_controller.py
@@ -389,9 +389,6 @@ def patch_connection(service_id, body=None):  # noqa: E501
         }
         return response, code
 
-    conn_status = ConnectionStateMachine.State.DOWN
-    body, _ = connection_state_machine(body, conn_status)
-
     logger.info(
         f"Modifying: Failed to place new connection. ID: {service_id} reason='{reason}', code={code}"
     )
diff --git a/sdx_controller/handlers/connection_handler.py b/sdx_controller/handlers/connection_handler.py
index c8a2bc78..f5601bed 100644
--- a/sdx_controller/handlers/connection_handler.py
+++ b/sdx_controller/handlers/connection_handler.py
@@ -555,7 +555,7 @@ def handle_link_failure(self, te_manager, failed_links):
 
                     logger.debug("Removed connection:")
                     logger.debug(connection)
-                    time.sleep(10)
+                    # time.sleep(10)
 
                     connection, _ = connection_state_machine(
                         connection, ConnectionStateMachine.State.RECOVERING
@@ -594,17 +594,20 @@ def handle_link_failure(self, te_manager, failed_links):
                             "status",
                             str(conn_status),
                         )
+                        _reason = (
+                            "place_connection failed during link failure rerouting"
+                        )
                         code = 400
 
                     logger.info(
                         f"place_connection result: ID: {service_id} reason='{_reason}', code={code}"
                     )
-                    response = {
-                        "service_id": service_id,
-                        "status": parse_conn_status(connection["status"]),
-                        "reason": _reason,
-                    }
-                    return response, code
+                    # response = {
+                    #    "service_id": service_id,
+                    #    "status": parse_conn_status(connection["status"]),
+                    #    "reason": _reason,
+                    # }
+                    # return response, code
 
     def handle_uni_ports_up_to_down(self, uni_ports_up_to_down):
         """

From e5fdd7e0dcd769c2a4f9553882d42f5683530296 Mon Sep 17 00:00:00 2001
From: Yufeng Xin <yufeng.xin@gmail.com>
Date: Fri, 24 Apr 2026 10:52:43 -0400
Subject: [PATCH 33/34] version 3.2.2

---
 README.md                                     | 23 +++++++++++--------
 pyproject.toml                                |  4 ++--
 sdx_controller/handlers/connection_handler.py |  1 -
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 0259151c..ea059c16 100644
--- a/README.md
+++ b/README.md
@@ -199,27 +199,30 @@ $ docker run --rm -d --name rabbitmq -p 5672:5672 -p 15672:15672 rabbitmq:latest
 $ docker run --rm -d --name mongo -p 27017:27017 -e MONGO_INITDB_ROOT_USERNAME=guest -e MONGO_INITDB_ROOT_PASSWORD=guest mongo:7.0.11
 ```
 
-Some environment variables are expected to be set for the tests to
-work as expected, so you may want to copy `env.template` to `.env` and
-edit it according to your environment, and make sure the env vars are
-present in your shell:
+Load environment variables from `env.template` directly into your shell
+(no need to copy it to `.env`), then override `MQ_HOST` and `MQ_PORT`
+to point at the local RabbitMQ container started above:
 
 ```console
-$ cp env.template .env 
-$ # and then edit .env to suit your environment
-$ source .env
+$ set -a; source env.template; set +a
+$ export MQ_HOST=localhost MQ_PORT=5672
 ```
 
-And now, activate a virtual environment, install the requirements, and
-then run `pytest`:
+Activate a virtual environment, install the requirements, and run `pytest`:
 
-```
+```console
 $ python3 -m venv venv --upgrade-deps
 $ source ./venv/bin/activate
 $ pip3 install --editable .[test]
 $ pytest
 ```
 
+To run a specific test file, pass its path to `pytest`:
+
+```console
+$ pytest sdx_controller/test/test_l2vpn_controller_patch.py
+```
+
 
 <!-- References -->
 
diff --git a/pyproject.toml b/pyproject.toml
index 2fe2192d..a3d81cc1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "sdx-controller"
-version = "3.2.0"
+version = "3.2.2"
 description = "AtlanticWave-SDX project's main controller"
 authors = [
     { name = "Yufeng Xin", email = "yxin@renci.org" },
@@ -29,7 +29,7 @@ dependencies = [
     "pika >= 1.2.0",
     "dataset",
     "pymongo > 3.0",
-    "sdx-pce @ git+https://github.com/atlanticwave-sdx/pce@302-port-nni-removed",
+    "sdx-pce @ git+https://github.com/atlanticwave-sdx/pce@v3.2.2",
 ]
 
 [project.optional-dependencies]
diff --git a/sdx_controller/handlers/connection_handler.py b/sdx_controller/handlers/connection_handler.py
index f5601bed..7c33921b 100644
--- a/sdx_controller/handlers/connection_handler.py
+++ b/sdx_controller/handlers/connection_handler.py
@@ -20,7 +20,6 @@
 )
 
 from sdx_controller.messaging.topic_queue_producer import TopicQueueProducer
-from sdx_controller.models import connection
 from sdx_controller.models.simple_link import SimpleLink
 from sdx_controller.utils.parse_helper import ParseHelper
 

From 69fb662ccfdca4147f87664d7c5e1cdea052c681 Mon Sep 17 00:00:00 2001
From: Cong Wang <cwang@renci.org>
Date: Fri, 24 Apr 2026 15:15:07 +0000
Subject: [PATCH 34/34] Initiate queue as durable

---
 sdx_controller/messaging/rpc_queue_consumer.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sdx_controller/messaging/rpc_queue_consumer.py b/sdx_controller/messaging/rpc_queue_consumer.py
index 299e58aa..a564d710 100644
--- a/sdx_controller/messaging/rpc_queue_consumer.py
+++ b/sdx_controller/messaging/rpc_queue_consumer.py
@@ -141,7 +141,10 @@ def __init__(self, thread_queue, exchange_name, te_manager):
         self.channel = self.connection.channel()
         self.exchange_name = exchange_name
 
-        self.channel.queue_declare(queue=SUB_QUEUE)
+        # RabbitMQ no longer permits transient non-exclusive queues by default.
+        # This shared controller queue should be durable so it remains compatible
+        # with newer broker defaults.
+        self.channel.queue_declare(queue=SUB_QUEUE, durable=True)
         self._thread_queue = thread_queue
 
         self.te_manager = te_manager