diff --git a/db/init/mysql/schema.sql b/db/init/mysql/schema.sql index 8891da9e819f..e91e3a1f5192 100644 --- a/db/init/mysql/schema.sql +++ b/db/init/mysql/schema.sql @@ -896,6 +896,19 @@ INSERT INTO `permission` VALUES ('1953049887387303922', '1346358560427216896', ' INSERT INTO `permission` VALUES ('1953049887387303923', '1346358560427216896', '1844026099075534903', '2023-08-31 06:59:01', '2023-08-31 06:59:01'); INSERT INTO `permission` VALUES ('1953049887387303924', '1346358560427216896', '1844026099075534904', '2023-08-31 06:59:01', '2023-08-31 06:59:01'); +-- httpRecord +INSERT INTO `permission` VALUES ('2060675900525121540', '1346358560427216896', '2060959853814128640', '2023-08-31 06:59:01', '2023-08-31 06:59:01'); +INSERT INTO `permission` VALUES ('2060675900525121541', '1346358560427216896', '2060553097071169537', '2023-08-31 07:22:07', '2023-08-31 07:22:07'); +INSERT INTO `permission` VALUES ('2060675900525121542', '1346358560427216896', '2060553097071169538', '2023-08-31 07:14:26', '2023-08-31 07:14:26'); +INSERT INTO `permission` VALUES ('2060675900525121543', '1346358560427216896', '2060553097071169539', '2023-08-31 07:22:07', '2023-08-31 07:22:07'); +INSERT INTO `permission` VALUES ('2060675900525121544', '1346358560427216896', '2060553097071169541', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121545', '1346358560427216896', '2060553097071169542', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121546', '1346358560427216896', '2060553097071169543', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121547', '1346358560427216896', '2060553097071169544', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121548', '1346358560427216896', '2060553097071169545', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121549', '1346358560427216896', '2060553097071169546', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121560', '1346358560427216896', '2060553097071169550', '2023-08-31 07:22:07', '2023-08-31 07:22:07'); + INSERT INTO `permission` VALUES ('1697146860569642741', '1346358560427216896', '1844026099075554850', '2023-08-31 06:59:01', '2023-08-31 06:59:01'); INSERT INTO `permission` VALUES ('1697146860569642742', '1346358560427216896', '1844026099075554851', '2023-08-31 07:22:07', '2023-08-31 07:22:07'); INSERT INTO `permission` VALUES ('1697146860569642743', '1346358560427216896', '1844026099075554852', '2023-08-31 07:14:26', '2023-08-31 07:14:26'); @@ -1019,6 +1032,7 @@ INSERT INTO `plugin` VALUES ('53', 'aiRequestTransformer', NULL, 'Ai', 65, 0, '2 INSERT INTO `plugin` VALUES ('61', 'mcpServer', null, 'MCP', 180, 0, '2023-12-20 18:02:53', '2023-12-20 18:02:53', null); INSERT INTO `plugin` VALUES ('66', 'aiResponseTransformer', NULL, 'Ai', 66, 0, '2023-12-20 18:02:53', '2023-12-20 18:02:53', null); +INSERT INTO `plugin` VALUES ('67', 'httpRecord', null, 'HttpProcess', 200, 1, '2023-12-20 18:02:53', '2026-05-24 17:19:17.859', NULL); -- ---------------------------- -- Table structure for plugin_handle @@ -1483,6 +1497,12 @@ INSERT INTO `plugin_handle` VALUES ('1722804548510507357', '66', 'apiKey', 'apiK INSERT INTO `plugin_handle` VALUES ('1722804548510507358', '66', 'model', 'model', 2, 2, 3, '{"required":"0","rule":""}', '2025-03-12 06:02:32.450', '2025-03-12 06:02:32.450'); INSERT INTO `plugin_handle` VALUES ('1722804548510507359', '66', 'content', 'content', 2, 2, 4, '{"required":"0","rule":""}', '2025-03-12 06:02:32.450', '2025-03-12 06:02:32.450'); +INSERT INTO `plugin_handle` VALUES('2067162479970447360', '67', 'startTime', 'startTime', 1, 2, 0, '{"required":"1","defaultValue":"1780287614000","placeholder":"startTime","rule":""}', '2026-06-17 16:28:35.904', '2026-06-17 16:29:13.623'); +INSERT INTO `plugin_handle` VALUES('2067162479970447361', '67', 'endTime', 'endTime', 1, 2, 0, '{"required":"1","defaultValue":"1780287814000","placeholder":"endTime","rule":""}', '2026-06-17 16:28:35.904', '2026-06-17 16:29:13.623'); +INSERT INTO `plugin_handle` VALUES('2068305908214779904', '67', 'maxBodySize', 'maxBodySize', 1, 3, 0, '{"required":"0","defaultValue":"524288","rule":""}', '2026-06-20 20:12:10.442', '2026-06-20 20:12:10.442'); +INSERT INTO `plugin_handle` VALUES('2068306022568284160', '67', 'batchSize', 'batchSize', 1, 3, 1, '{"required":"0","defaultValue":"100","rule":""}', '2026-06-20 20:12:37.705', '2026-06-20 20:14:03.068'); +INSERT INTO `plugin_handle` VALUES('2068306133750894592', '67', 'batchIntervalMs', 'batchIntervalMs', 1, 3, 0, '{"required":"0","defaultValue":"5000","rule":""}', '2026-06-20 20:13:04.213', '2026-06-20 20:13:39.534'); + -- ---------------------------- -- Table structure for resource -- ---------------------------- @@ -2080,6 +2100,20 @@ INSERT INTO `resource` VALUES ('1844026099075565007', '1844026099075565000', 'SH INSERT INTO `resource` VALUES ('1844026099075565008', '1844026099075565000', 'SHENYU.BUTTON.PLUGIN.RULE.DELETE', '', '', '', 2, 0, '', 1, 0, 'plugin:aiResponseTransformerRule:delete', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); INSERT INTO `resource` VALUES ('1844026099075565009', '1844026099075565000', 'SHENYU.BUTTON.PLUGIN.SYNCHRONIZE', '', '', '', 2, 0, '', 1, 0, 'plugin:aiResponseTransformer:modify', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +-- httpRecord +INSERT INTO `resource` VALUES ('2060959853814128640', '1346775491550474240', 'httpRecord', 'httpRecord', '/plug/httpRecord', 'httpRecord', 1, 0, 'key', 0, 0, '', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169537', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.ADD', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:add', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169538', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.QUERY', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:query', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169539', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.EDIT', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:edit', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169541', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.DELETE', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:delete', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169542', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.ADD', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:add', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169543', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.QUERY', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:query', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169544', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.EDIT', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:edit', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169545', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.DELETE', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:delete', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169546', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SYNCHRONIZE', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecord:modify', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169550', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.LIST', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecord:list', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); + + -- ---------------------------- -- Table structure for role -- ---------------------------- @@ -2642,6 +2676,7 @@ INSERT INTO `namespace_plugin_rel` (`id`,`namespace_id`,`plugin_id`, `config`, ` INSERT INTO `namespace_plugin_rel` (`id`,`namespace_id`,`plugin_id`, `config`, `sort`, `enabled`, `date_created`, `date_updated`) VALUES ('1801816010882832189','649330b6-c2d7-4edc-be8e-8a54df9eb385','61', NULL, 180, 0, '2022-05-25 18:02:53.000', '2022-05-25 18:02:53.000'); INSERT INTO `namespace_plugin_rel` (`id`,`namespace_id`,`plugin_id`, `config`, `sort`, `enabled`, `date_created`, `date_updated`) VALUES ('1801816010882822300','649330b6-c2d7-4edc-be8e-8a54df9eb385','66', NULL, 66, 0, '2022-05-25 18:02:53.000', '2022-05-25 18:02:53.000'); +INSERT INTO `namespace_plugin_rel` (`id`,`namespace_id`,`plugin_id`, `config`, `sort`, `enabled`, `date_created`, `date_updated`) VALUES ('1801816010882822191','649330b6-c2d7-4edc-be8e-8a54df9eb385','67', NULL, 199,1, '2022-05-25 18:02:53.000', '2026-05-25 09:27:25.639'); INSERT INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`, `date_created`, `date_updated`) VALUES ('1792749362445840479', '1357956838021890048', 'SHENYU.MENU.SYSTEM.MANAGMENT.NAMESPACEPLUGIN', 'namespacePlugin', '/config/namespacePlugin', 'namespacePlugin', 1, 2, 'build', 0, 0, '', 1, '2024-06-25 18:02:53.000', '2024-06-25 18:02:53.000'); diff --git a/db/init/ob/schema.sql b/db/init/ob/schema.sql index d2fdbc7cb125..43e03ddd16a3 100644 --- a/db/init/ob/schema.sql +++ b/db/init/ob/schema.sql @@ -922,6 +922,18 @@ INSERT INTO `permission` VALUES ('1697146861569542757', '1346358560427216896', ' INSERT INTO `permission` VALUES ('1697146861569542758', '1346358560427216896', '1844026199075534867', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); INSERT INTO `permission` VALUES ('1697146861569542759', '1346358560427216896', '1844026199075534868', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); INSERT INTO `permission` VALUES ('1697146861569542760', '1346358560427216896', '1844026199075534869', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); + +INSERT INTO `permission` VALUES ('2060675900525121540', '1346358560427216896', '2060959853814128640', '2023-08-31 06:59:01', '2023-08-31 06:59:01'); +INSERT INTO `permission` VALUES ('2060675900525121541', '1346358560427216896', '2060553097071169537', '2023-08-31 07:22:07', '2023-08-31 07:22:07'); +INSERT INTO `permission` VALUES ('2060675900525121542', '1346358560427216896', '2060553097071169538', '2023-08-31 07:14:26', '2023-08-31 07:14:26'); +INSERT INTO `permission` VALUES ('2060675900525121543', '1346358560427216896', '2060553097071169539', '2023-08-31 07:22:07', '2023-08-31 07:22:07'); +INSERT INTO `permission` VALUES ('2060675900525121544', '1346358560427216896', '2060553097071169541', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121545', '1346358560427216896', '2060553097071169542', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121546', '1346358560427216896', '2060553097071169543', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121547', '1346358560427216896', '2060553097071169544', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121548', '1346358560427216896', '2060553097071169545', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121549', '1346358560427216896', '2060553097071169546', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121560', '1346358560427216896', '2060553097071169550', '2023-08-31 07:22:07', '2023-08-31 07:22:07'); -- ---------------------------- -- Table structure for plugin -- ---------------------------- @@ -987,6 +999,8 @@ INSERT INTO `plugin` VALUES ('51', 'aiTokenLimiter', NULL, 'Ai', 171, 0, '2023-1 INSERT INTO `plugin` VALUES ('53', 'aiRequestTransformer', NULL, 'Ai', 65, 0, '2023-12-20 18:02:53', '2023-12-20 18:02:53', null); INSERT INTO `plugin` VALUES ('61', 'mcpServer', null, 'MCP', 180, 0, '2023-12-20 18:02:53', '2023-12-20 18:02:53', null); INSERT INTO `plugin` VALUES ('66', 'aiResponseTransformer', NULL, 'Ai', 66, 0, '2023-12-20 18:02:53', '2023-12-20 18:02:53', null); +INSERT INTO `plugin` VALUES ('67', 'httpRecord', null, 'HttpProcess', 200, 1, '2023-12-20 18:02:53', '2026-05-24 17:19:17.859', NULL); + -- ---------------------------- -- Table structure for plugin_handle -- ---------------------------- @@ -2010,6 +2024,18 @@ INSERT INTO `resource` VALUES ('1844026099075565006', '1844026099075565000', 'SH INSERT INTO `resource` VALUES ('1844026099075565007', '1844026099075565000', 'SHENYU.BUTTON.PLUGIN.RULE.EDIT', '', '', '', 2, 0, '', 1, 0, 'plugin:aiResponseTransformerRule:edit', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); INSERT INTO `resource` VALUES ('1844026099075565008', '1844026099075565000', 'SHENYU.BUTTON.PLUGIN.RULE.DELETE', '', '', '', 2, 0, '', 1, 0, 'plugin:aiResponseTransformerRule:delete', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); INSERT INTO `resource` VALUES ('1844026099075565009', '1844026099075565000', 'SHENYU.BUTTON.PLUGIN.SYNCHRONIZE', '', '', '', 2, 0, '', 1, 0, 'plugin:aiResponseTransformer:modify', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); + +INSERT INTO `resource` VALUES ('2060959853814128640', '1346775491550474240', 'httpRecord', 'httpRecord', '/plug/httpRecord', 'httpRecord', 1, 0, 'key', 0, 0, '', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169537', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.ADD', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:add', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169538', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.QUERY', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:query', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169539', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.EDIT', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:edit', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169541', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.DELETE', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:delete', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169542', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.ADD', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:add', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169543', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.QUERY', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:query', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169544', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.EDIT', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:edit', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169545', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.DELETE', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:delete', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169546', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SYNCHRONIZE', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecord:modify', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169550', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.LIST', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecord:list', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); -- ---------------------------- -- Table structure for role -- ---------------------------- @@ -2564,6 +2590,7 @@ INSERT INTO `namespace_plugin_rel` (`id`,`namespace_id`,`plugin_id`, `config`, ` INSERT INTO `namespace_plugin_rel` (`id`,`namespace_id`,`plugin_id`, `config`, `sort`, `enabled`, `date_created`, `date_updated`) VALUES ('1801816010882822188','649330b6-c2d7-4edc-be8e-8a54df9eb385','51', NULL, 171, 0, '2022-05-25 18:02:53.000', '2022-05-25 18:02:53.000'); INSERT INTO `namespace_plugin_rel` (`id`,`namespace_id`,`plugin_id`, `config`, `sort`, `enabled`, `date_created`, `date_updated`) VALUES ('1801816010882832189','649330b6-c2d7-4edc-be8e-8a54df9eb385','61', NULL, 180, 0, '2022-05-25 18:02:53.000', '2022-05-25 18:02:53.000'); INSERT INTO `namespace_plugin_rel` (`id`,`namespace_id`,`plugin_id`, `config`, `sort`, `enabled`, `date_created`, `date_updated`) VALUES ('1801816010882822300','649330b6-c2d7-4edc-be8e-8a54df9eb385','66', NULL, 66, 0, '2022-05-25 18:02:53.000', '2022-05-25 18:02:53.000'); +INSERT INTO `namespace_plugin_rel` (`id`,`namespace_id`,`plugin_id`, `config`, `sort`, `enabled`, `date_created`, `date_updated`) VALUES ('1801816010882822191','649330b6-c2d7-4edc-be8e-8a54df9eb385','67', NULL, 199,1, '2022-05-25 18:02:53.000', '2026-05-25 09:27:25.639'); INSERT INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`, `date_created`, `date_updated`) VALUES ('1792749362445840479', '1357956838021890048', 'SHENYU.MENU.SYSTEM.MANAGMENT.NAMESPACEPLUGIN', 'namespacePlugin', '/config/namespacePlugin', 'namespacePlugin', 1, 2, 'build', 0, 0, '', 1, '2024-06-25 18:02:53.000', '2024-06-25 18:02:53.000'); INSERT INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`, `date_created`, `date_updated`) VALUES ('1792749362445840480', '1792749362445840479', 'SHENYU.BUTTON.SYSTEM.LIST', '', '', '', 2, 0, '', 1, 0, 'system:namespacePlugin:list', 1, '2024-06-25 18:02:53.000', '2024-06-25 18:02:53.000'); @@ -2671,6 +2698,11 @@ INSERT INTO `plugin_handle` VALUES ('1722804548510507356', '66', 'baseUrl', 'bas INSERT INTO `plugin_handle` VALUES ('1722804548510507357', '66', 'apiKey', 'apiKey', 2, 2, 2, '{"required":"0","rule":""}', '2025-03-12 06:02:18.707', '2025-03-12 06:02:18.707'); INSERT INTO `plugin_handle` VALUES ('1722804548510507358', '66', 'model', 'model', 2, 2, 3, '{"required":"0","rule":""}', '2025-03-12 06:02:32.450', '2025-03-12 06:02:32.450'); INSERT INTO `plugin_handle` VALUES ('1722804548510507359', '66', 'content', 'content', 2, 2, 4, '{"required":"0","rule":""}', '2025-03-12 06:02:32.450', '2025-03-12 06:02:32.450'); +INSERT INTO `plugin_handle` VALUES ('2067162479970447360', '67', 'startTime', 'startTime', 1, 2, 0, '{"required":"1","defaultValue":"1780287614000","placeholder":"startTime","rule":""}', '2026-06-17 16:28:35.904', '2026-06-17 16:29:13.623'); +INSERT INTO `plugin_handle` VALUES ('2067162479970447361', '67', 'endTime', 'endTime', 1, 2, 0, '{"required":"1","defaultValue":"1780287814000","placeholder":"endTime","rule":""}', '2026-06-17 16:28:35.904', '2026-06-17 16:29:13.623'); +INSERT INTO `plugin_handle` VALUES ('2068305908214779904', '67', 'maxBodySize', 'maxBodySize', 1, 3, 0, '{"required":"0","defaultValue":"524288","rule":""}', '2026-06-20 20:12:10.442', '2026-06-20 20:12:10.442'); +INSERT INTO `plugin_handle` VALUES ('2068306022568284160', '67', 'batchSize', 'batchSize', 1, 3, 1, '{"required":"0","defaultValue":"100","rule":""}', '2026-06-20 20:12:37.705', '2026-06-20 20:14:03.068'); +INSERT INTO `plugin_handle` VALUES ('2068306133750894592', '67', 'batchIntervalMs', 'batchIntervalMs', 1, 3, 0, '{"required":"0","defaultValue":"5000","rule":""}', '2026-06-20 20:13:04.213', '2026-06-20 20:13:39.534'); DROP TABLE IF EXISTS `registry_config`; CREATE TABLE `registry_config` ( diff --git a/db/upgrade/2.7.0-upgrade-2.7.1-mysql.sql b/db/upgrade/2.7.0-upgrade-2.7.1-mysql.sql index 978b4b53e749..e80e1ba6cd0a 100755 --- a/db/upgrade/2.7.0-upgrade-2.7.1-mysql.sql +++ b/db/upgrade/2.7.0-upgrade-2.7.1-mysql.sql @@ -360,3 +360,38 @@ DELETE FROM `namespace_plugin_rel` WHERE `plugin_id` = '17'; DELETE FROM `plugin_handle` WHERE `plugin_id` = '17'; DELETE FROM `plugin` WHERE `id` = '17'; ALTER TABLE `api` MODIFY COLUMN `rpc_type` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'http,dubbo,sofa,tars,websocket,springCloud,grpc'; + +-- httpRecord +INSERT INTO `plugin` VALUES ('67', 'httpRecord', null, 'HttpProcess', 200, 1, '2023-12-20 18:02:53', '2026-05-24 17:19:17.859', NULL); + +INSERT INTO `plugin_handle` VALUES ('2067162479970447360', '67', 'startTime', 'startTime', 1, 2, 0, '{"required":"1","defaultValue":"1780287614000","placeholder":"startTime","rule":""}', '2026-06-17 16:28:35.904', '2026-06-17 16:29:13.623'); +INSERT INTO `plugin_handle` VALUES ('2067162479970447361', '67', 'endTime', 'endTime', 1, 2, 0, '{"required":"1","defaultValue":"1780287814000","placeholder":"endTime","rule":""}', '2026-06-17 16:28:35.904', '2026-06-17 16:29:13.623'); +INSERT INTO `plugin_handle` VALUES ('2068305908214779904', '67', 'maxBodySize', 'maxBodySize', 1, 3, 0, '{"required":"0","defaultValue":"524288","rule":""}', '2026-06-20 20:12:10.442', '2026-06-20 20:12:10.442'); +INSERT INTO `plugin_handle` VALUES ('2068306022568284160', '67', 'batchSize', 'batchSize', 1, 3, 1, '{"required":"0","defaultValue":"100","rule":""}', '2026-06-20 20:12:37.705', '2026-06-20 20:14:03.068'); +INSERT INTO `plugin_handle` VALUES ('2068306133750894592', '67', 'batchIntervalMs', 'batchIntervalMs', 1, 3, 0, '{"required":"0","defaultValue":"5000","rule":""}', '2026-06-20 20:13:04.213', '2026-06-20 20:13:39.534'); + +INSERT INTO `namespace_plugin_rel` (`id`,`namespace_id`,`plugin_id`, `config`, `sort`, `enabled`, `date_created`, `date_updated`) VALUES ('1801816010882822191','649330b6-c2d7-4edc-be8e-8a54df9eb385','67', NULL, 199,1, '2022-05-25 18:02:53.000', '2026-05-25 09:27:25.639'); + +INSERT INTO `permission` VALUES ('2060675900525121540', '1346358560427216896', '2060959853814128640', '2023-08-31 06:59:01', '2023-08-31 06:59:01'); +INSERT INTO `permission` VALUES ('2060675900525121541', '1346358560427216896', '2060553097071169537', '2023-08-31 07:22:07', '2023-08-31 07:22:07'); +INSERT INTO `permission` VALUES ('2060675900525121542', '1346358560427216896', '2060553097071169538', '2023-08-31 07:14:26', '2023-08-31 07:14:26'); +INSERT INTO `permission` VALUES ('2060675900525121543', '1346358560427216896', '2060553097071169539', '2023-08-31 07:22:07', '2023-08-31 07:22:07'); +INSERT INTO `permission` VALUES ('2060675900525121544', '1346358560427216896', '2060553097071169541', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121545', '1346358560427216896', '2060553097071169542', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121546', '1346358560427216896', '2060553097071169543', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121547', '1346358560427216896', '2060553097071169544', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121548', '1346358560427216896', '2060553097071169545', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121549', '1346358560427216896', '2060553097071169546', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT INTO `permission` VALUES ('2060675900525121560', '1346358560427216896', '2060553097071169550', '2023-08-31 07:22:07', '2023-08-31 07:22:07'); + +INSERT INTO `resource` VALUES ('2060959853814128640', '1346775491550474240', 'httpRecord', 'httpRecord', '/plug/httpRecord', 'httpRecord', 1, 0, 'key', 0, 0, '', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169537', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.ADD', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:add', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169538', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.QUERY', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:query', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169539', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.EDIT', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:edit', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169541', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.DELETE', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:delete', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169542', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.ADD', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:add', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169543', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.QUERY', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:query', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169544', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.EDIT', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:edit', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169545', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.DELETE', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:delete', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169546', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SYNCHRONIZE', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecord:modify', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); +INSERT INTO `resource` VALUES ('2060553097071169550', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.LIST', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecord:list', 1, '2022-05-25 18:02:58', '2022-05-25 18:02:58'); diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/HttpRecordConfiguration.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/HttpRecordConfiguration.java new file mode 100644 index 000000000000..c5be51f5118b --- /dev/null +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/HttpRecordConfiguration.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.config; + +import org.apache.shenyu.admin.config.properties.HttpRecordProperties; +import org.apache.shenyu.admin.disruptor.ShenyuHttpRequestRecordDisruptorPublisher; +import org.apache.shenyu.admin.record.HttpRecordRepository; +import org.apache.shenyu.common.concurrent.MemorySafeTaskQueue; +import org.apache.shenyu.common.concurrent.ShenyuThreadFactory; +import org.apache.shenyu.common.concurrent.ShenyuThreadPoolExecutor; +import org.apache.shenyu.common.config.ShenyuConfig; +import org.apache.shenyu.common.constant.Constants; +import org.apache.shenyu.common.utils.Singleton; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +@Configuration +public class HttpRecordConfiguration { + + @Bean + public ShenyuHttpRequestRecordDisruptorPublisher shenyuHttpRequestRecordDisruptorPublisher( + final HttpRecordProperties properties, + final Map repositoryMap) { + ShenyuHttpRequestRecordDisruptorPublisher publisher = new ShenyuHttpRequestRecordDisruptorPublisher(); + publisher.start(properties, repositoryMap); + return publisher; + } + + @Bean(name = "replayExecutor") + public ShenyuThreadPoolExecutor replayExecutor() { + ShenyuConfig config = Optional.ofNullable(Singleton.INST.get(ShenyuConfig.class)).orElse(new ShenyuConfig()); + final ShenyuConfig.SharedPool sharedPool = config.getSharedPool(); + return new ShenyuThreadPoolExecutor(sharedPool.getCorePoolSize(), + sharedPool.getMaximumPoolSize(), sharedPool.getKeepAliveTime(), TimeUnit.MILLISECONDS, + new MemorySafeTaskQueue<>(Constants.THE_256_MB), + ShenyuThreadFactory.create(config.getSharedPool().getPrefix(), true), + new ThreadPoolExecutor.AbortPolicy()); + } + +} diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/properties/HttpRecordProperties.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/properties/HttpRecordProperties.java new file mode 100644 index 000000000000..8be9753c157b --- /dev/null +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/config/properties/HttpRecordProperties.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.config.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties(prefix = "shenyu.http-record") +public class HttpRecordProperties { + + private String storageType; + + private String storagePath; + + public String getStorageType() { + return storageType; + } + + public void setStorageType(final String storageType) { + this.storageType = storageType; + } + + public String getStoragePath() { + return storagePath; + } + + public void setStoragePath(final String storagePath) { + this.storagePath = storagePath; + } +} diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/controller/HttpRecordController.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/controller/HttpRecordController.java new file mode 100644 index 000000000000..3475d4ed4607 --- /dev/null +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/controller/HttpRecordController.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.controller; + +import org.apache.commons.collections4.CollectionUtils; +import jakarta.annotation.Resource; +import org.apache.commons.lang3.StringUtils; +import org.apache.shenyu.admin.aspect.annotation.RestApi; +import org.apache.shenyu.admin.disruptor.ShenyuHttpRequestRecordDisruptorPublisher; +import org.apache.shenyu.admin.model.dto.ShenyuHttpRequestRecordDTO; +import org.apache.shenyu.admin.model.result.ShenyuAdminResult; +import org.apache.shenyu.admin.record.entity.ReplayProgress; +import org.apache.shenyu.admin.service.HttpRecordService; +import org.apache.shenyu.admin.utils.ShenyuResultMessage; +import org.apache.shenyu.common.constant.Constants; +import org.apache.shenyu.common.utils.GsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestParam; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Objects; +import java.util.zip.CRC32; + +@RestApi("/record") +public class HttpRecordController { + + private static final Logger LOG = LoggerFactory.getLogger(HttpRecordController.class); + + @Resource + private ShenyuHttpRequestRecordDisruptorPublisher publisher; + + @Resource + private HttpRecordService httpRecordService; + + @PostMapping("/upload-record") + public String upload( + @RequestHeader(value = Constants.X_RECORD_CRC32) final Long clientCrc32, + @RequestBody final byte[] recordBytes) { + try { + if (Objects.nonNull(clientCrc32)) { + CRC32 crc32 = new CRC32(); + crc32.update(recordBytes); + long serverCrc32 = crc32.getValue(); + if (serverCrc32 != clientCrc32) { + LOG.warn("Record CRC32 validation failed! Client: {}, Server: {}. Data dropped silently.", clientCrc32, serverCrc32); + return ShenyuResultMessage.SUCCESS; + } + } + String recordBody = new String(recordBytes, StandardCharsets.UTF_8); + List records = GsonUtils.getInstance().fromList( + recordBody, + ShenyuHttpRequestRecordDTO.Record.class + ); + if (CollectionUtils.isNotEmpty(records)) { + ShenyuHttpRequestRecordDTO dto = new ShenyuHttpRequestRecordDTO(); + dto.setRecords(records); + publisher.publish(dto); + } + return ShenyuResultMessage.SUCCESS; + } catch (Exception e) { + LOG.warn("Failed to parse record body, data dropped silently. Reason: {}", e.getMessage()); + return ShenyuResultMessage.SUCCESS; + } + } + + /** + * List all available record tasks that can be replayed. + * + * @return list of record task metadata + */ + @GetMapping("") + public ShenyuAdminResult listRecordTasks() { + return ShenyuAdminResult.success(httpRecordService.listRecordTasks()); + } + + /** + * Start an asynchronous replay of all records belonging to the given taskId. + * + * @param taskId maps to {storagePath}/{taskId}.jsonl + * @param targetHost base URL of the gateway to replay against, e.g. http://127.0.0.1:9195 + * @return replayId for polling progress + */ + @PostMapping("/replay") + public ShenyuAdminResult startReplay( + @RequestParam final String taskId, + @RequestParam final String targetHost) { + if (StringUtils.isAnyBlank(taskId, targetHost)) { + return ShenyuAdminResult.error("taskId and targetHost must not be blank"); + } + try { + String replayId = httpRecordService.startReplay(taskId, targetHost); + return ShenyuAdminResult.success(null, replayId); + } catch (IllegalArgumentException e) { + return ShenyuAdminResult.error(e.getMessage()); + } catch (Exception e) { + LOG.error("Failed to start replay. taskId={}, targetHost={}", taskId, targetHost, e); + return ShenyuAdminResult.error("Failed to start replay: " + e.getMessage()); + } + } + + /** + * Poll the progress of an ongoing or completed replay job. + * + * @param replayId the id returned by {@link #startReplay} + * @return current progress snapshot + */ + @GetMapping("/progress/{replayId}") + public ShenyuAdminResult getReplayProgress(@PathVariable final String replayId) { + ReplayProgress progress = httpRecordService.getReplayProgress(replayId); + if (Objects.isNull(progress)) { + return ShenyuAdminResult.error("Replay not found for replayId: " + replayId); + } + return ShenyuAdminResult.success(progress); + } +} diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/disruptor/ShenyuHttpRequestRecordDisruptorPublisher.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/disruptor/ShenyuHttpRequestRecordDisruptorPublisher.java new file mode 100644 index 000000000000..faa18f07e397 --- /dev/null +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/disruptor/ShenyuHttpRequestRecordDisruptorPublisher.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.disruptor; + +import org.apache.shenyu.admin.config.properties.HttpRecordProperties; +import org.apache.shenyu.admin.disruptor.executor.ShenyuHttpRequestRecordConsumerExecutor.ShenyuHttpRequestRecordConsumerExecutorFactory; +import org.apache.shenyu.admin.model.dto.ShenyuHttpRequestRecordDTO; +import org.apache.shenyu.admin.record.HttpRecordRepository; +import org.apache.shenyu.disruptor.DisruptorProviderManage; +import org.apache.shenyu.disruptor.provider.DisruptorProvider; + +import java.util.Map; + +public class ShenyuHttpRequestRecordDisruptorPublisher { + + private DisruptorProviderManage providerManage; + + + public void start(final HttpRecordProperties properties, final Map repositoryMap) { + ShenyuHttpRequestRecordConsumerExecutorFactory factory = new ShenyuHttpRequestRecordConsumerExecutorFactory(properties, repositoryMap); + providerManage = new DisruptorProviderManage<>(factory); + providerManage.startup(); + } + + public void publish(final ShenyuHttpRequestRecordDTO dto) { + DisruptorProvider provider = providerManage.getProvider(); + provider.onData(dto); + } + + +} diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/disruptor/executor/ShenyuHttpRequestRecordConsumerExecutor.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/disruptor/executor/ShenyuHttpRequestRecordConsumerExecutor.java new file mode 100644 index 000000000000..ca40a7817398 --- /dev/null +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/disruptor/executor/ShenyuHttpRequestRecordConsumerExecutor.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.disruptor.executor; + +import org.apache.commons.lang3.StringUtils; +import org.apache.shenyu.admin.config.properties.HttpRecordProperties; +import org.apache.shenyu.admin.model.dto.ShenyuHttpRequestRecordDTO; +import org.apache.shenyu.admin.record.HttpRecordRepository; +import org.apache.shenyu.disruptor.consumer.QueueConsumerExecutor; +import org.apache.shenyu.disruptor.consumer.QueueConsumerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +public class ShenyuHttpRequestRecordConsumerExecutor extends QueueConsumerExecutor { + + private static final Logger LOG = LoggerFactory.getLogger(ShenyuHttpRequestRecordConsumerExecutor.class); + + private final HttpRecordProperties properties; + + private final Map repositoryMap; + + public ShenyuHttpRequestRecordConsumerExecutor(final HttpRecordProperties properties, final Map repositoryMap) { + this.properties = properties; + this.repositoryMap = repositoryMap; + } + + @Override + public void run() { + ShenyuHttpRequestRecordDTO dto = getData(); + String storageType = StringUtils.isNotBlank(properties.getStorageType()) ? properties.getStorageType() : "local"; + HttpRecordRepository repository = repositoryMap.getOrDefault(storageType, repositoryMap.get("local")); + repository.save(dto); + } + + public static class ShenyuHttpRequestRecordConsumerExecutorFactory implements QueueConsumerFactory { + + private final HttpRecordProperties properties; + + private final Map repositoryMap; + + public ShenyuHttpRequestRecordConsumerExecutorFactory(final HttpRecordProperties properties, final Map repositoryMap) { + this.properties = properties; + this.repositoryMap = repositoryMap; + } + + @Override + public QueueConsumerExecutor create() { + return new ShenyuHttpRequestRecordConsumerExecutor(properties, repositoryMap); + } + + + @Override + public String fixName() { + return "shenyu_http_request_record"; + } + + } +} diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/model/dto/ShenyuHttpRequestRecordDTO.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/model/dto/ShenyuHttpRequestRecordDTO.java new file mode 100644 index 000000000000..9525f4425a6d --- /dev/null +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/model/dto/ShenyuHttpRequestRecordDTO.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.model.dto; + +import java.util.List; +import java.util.Map; + +public class ShenyuHttpRequestRecordDTO { + + private List records; + + public List getRecords() { + return records; + } + + public void setRecords(final List records) { + this.records = records; + } + + public static class Record { + + private String taskId; + + private String traceId; + + private String method; + + private String requestUri; + + private String queryParams; + + private String requestBody; + + private Map requestHeaders; + + private Integer status; + + private Map responseHeaders; + + private String responseBody; + + public String getTaskId() { + return taskId; + } + + public void setTaskId(final String taskId) { + this.taskId = taskId; + } + + public void setRequestBody(final String requestBody) { + this.requestBody = requestBody; + } + + public void setMethod(final String method) { + this.method = method; + } + + public void setRequestUri(final String requestUri) { + this.requestUri = requestUri; + } + + public void setQueryParams(final String queryParams) { + this.queryParams = queryParams; + } + + public void setRequestHeaders(final Map requestHeaders) { + this.requestHeaders = requestHeaders; + } + + public void setStatus(final Integer status) { + this.status = status; + } + + public void setResponseHeaders(final Map responseHeaders) { + this.responseHeaders = responseHeaders; + } + + public void setResponseBody(final String responseBody) { + this.responseBody = responseBody; + } + + public void setTraceId(final String traceId) { + this.traceId = traceId; + } + + public String getTraceId() { + return traceId; + } + + public String getMethod() { + return method; + } + + public String getRequestUri() { + return requestUri; + } + + public String getQueryParams() { + return queryParams; + } + + public Map getRequestHeaders() { + return requestHeaders; + } + + public Integer getStatus() { + return status; + } + + public String getResponseBody() { + return responseBody; + } + + public Map getResponseHeaders() { + return responseHeaders; + } + + public String getRequestBody() { + return requestBody; + } + + } + +} diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/record/HttpRecordRepository.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/record/HttpRecordRepository.java new file mode 100644 index 000000000000..25ce34eb8d4f --- /dev/null +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/record/HttpRecordRepository.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.record; + +import org.apache.shenyu.admin.model.dto.ShenyuHttpRequestRecordDTO; +import org.apache.shenyu.admin.record.entity.RecordTaskInfo; + +import java.util.List; + +public interface HttpRecordRepository { + + void save(ShenyuHttpRequestRecordDTO dto); + + List listRecordTasks(); + + List loadRecords(String taskId); + +} diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/record/LocalJsonFileHttpRecordRepository.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/record/LocalJsonFileHttpRecordRepository.java new file mode 100644 index 000000000000..66d18777bc8e --- /dev/null +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/record/LocalJsonFileHttpRecordRepository.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.record; + +import org.apache.commons.lang3.StringUtils; +import org.apache.shenyu.admin.config.properties.HttpRecordProperties; +import org.apache.shenyu.admin.model.dto.ShenyuHttpRequestRecordDTO; +import org.apache.shenyu.admin.record.entity.RecordTaskInfo; +import org.apache.shenyu.common.utils.GsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Component("local") +public class LocalJsonFileHttpRecordRepository implements HttpRecordRepository { + + private static final Logger LOG = LoggerFactory.getLogger(LocalJsonFileHttpRecordRepository.class); + + private final HttpRecordProperties properties; + + public LocalJsonFileHttpRecordRepository(final HttpRecordProperties properties) { + this.properties = properties; + } + + @Override + public void save(final ShenyuHttpRequestRecordDTO dto) { + + String storagePath = properties.getStoragePath(); + + List records = dto.getRecords(); + Map> groupedMap = records.stream() + .collect(Collectors.groupingBy(ShenyuHttpRequestRecordDTO.Record::getTaskId)); + groupedMap.forEach((taskId, recordList) -> { + try { + StringBuilder chunkBuffer = new StringBuilder(1024 * 100); + for (ShenyuHttpRequestRecordDTO.Record record : recordList) { + String jsonLine = GsonUtils.getInstance().toJson(record); + chunkBuffer.append(jsonLine).append(System.lineSeparator()); + } + Path dirPath = Paths.get(storagePath); + String fileName = String.format("%s.jsonl", taskId); + Path targetFile = dirPath.resolve(fileName); + if (!Files.exists(targetFile)) { + Files.createDirectories(targetFile.getParent()); + } + Files.writeString(targetFile, chunkBuffer.toString(), + StandardOpenOption.CREATE, + StandardOpenOption.APPEND); + } catch (Exception e) { + LOG.error("storage httpRecord fail, storagePath:{}", storagePath, e); + } + }); + } + + @Override + public List listRecordTasks() { + Path storageDir = Paths.get(properties.getStoragePath()); + LOG.info("Listing record tasks from storage directory: {}", storageDir); + List result = new ArrayList<>(); + if (!Files.exists(storageDir) || !Files.isDirectory(storageDir)) { + return result; + } + try (Stream paths = Files.list(storageDir)) { + paths.filter(p -> p.getFileName().toString().endsWith(".jsonl")) + .forEach(p -> { + String fileName = p.getFileName().toString(); + String taskId = fileName.substring(0, fileName.length() - ".jsonl".length()); + long recordCount = countLines(p); + try { + long lastModified = Files.getLastModifiedTime(p).toMillis(); + result.add(new RecordTaskInfo(taskId, recordCount, lastModified)); + } catch (IOException e) { + LOG.warn("Failed to read record file metadata: {}", p, e); + } + }); + } catch (IOException e) { + LOG.error("Failed to list record files in: {}", storageDir, e); + } + return result; + } + + @Override + public List loadRecords(final String taskId) { + if (StringUtils.isBlank(taskId) || !taskId.matches("^[a-zA-Z0-9_-]+$")) { + LOG.warn("Invalid taskId: {}", taskId); + return new ArrayList<>(); + } + Path filePath = Paths.get(properties.getStoragePath(), taskId + ".jsonl").normalize(); + if (!filePath.startsWith(Paths.get(properties.getStoragePath()).normalize())) { + LOG.warn("Path traversal detected: {}", taskId); + return new ArrayList<>(); + } + List records = new ArrayList<>(); + if (!Files.exists(filePath)) { + LOG.warn("Record file not found: {}", filePath); + return records; + } + try (Stream lines = Files.lines(filePath)) { + lines.filter(StringUtils::isNotBlank) + .map(line -> GsonUtils.getInstance().fromJson(line, ShenyuHttpRequestRecordDTO.Record.class)) + .filter(Objects::nonNull) + .forEach(records::add); + } catch (IOException e) { + LOG.error("Failed to read record file: {}", filePath, e); + } + return records; + } + + private long countLines(final Path filePath) { + try (Stream lines = Files.lines(filePath)) { + return lines.filter(StringUtils::isNotBlank).count(); + } catch (IOException e) { + LOG.warn("Failed to count lines in: {}", filePath, e); + return 0; + } + } + +} diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/record/entity/RecordTaskInfo.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/record/entity/RecordTaskInfo.java new file mode 100644 index 000000000000..6632d5b32aea --- /dev/null +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/record/entity/RecordTaskInfo.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.record.entity; + +public class RecordTaskInfo { + + private final String taskId; + + private final long recordCount; + + private final long lastModifiedMs; + + public RecordTaskInfo(final String taskId, final long recordCount, final long lastModifiedMs) { + this.taskId = taskId; + this.recordCount = recordCount; + this.lastModifiedMs = lastModifiedMs; + } + + /** + * Get task id. + * + * @return taskId + */ + public String getTaskId() { + return taskId; + } + + /** + * Get the number of records in this task file. + * + * @return recordCount + */ + public long getRecordCount() { + return recordCount; + } + + /** + * Get last modified time in epoch milliseconds. + * + * @return lastModifiedMs + */ + public long getLastModifiedMs() { + return lastModifiedMs; + } +} + diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/record/entity/ReplayProgress.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/record/entity/ReplayProgress.java new file mode 100644 index 000000000000..61e8e00f11ee --- /dev/null +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/record/entity/ReplayProgress.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.record.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.concurrent.atomic.AtomicInteger; + +public class ReplayProgress { + + private final String replayId; + + private final String taskId; + + private final int total; + + private final AtomicInteger succeeded = new AtomicInteger(0); + + private final AtomicInteger failed = new AtomicInteger(0); + + public ReplayProgress(final String replayId, final String taskId, final int total) { + this.replayId = replayId; + this.taskId = taskId; + this.total = total; + } + + public void incrementSucceeded() { + succeeded.incrementAndGet(); + } + + public void incrementFailed() { + failed.incrementAndGet(); + } + + public String getReplayId() { + return replayId; + } + + public String getTaskId() { + return taskId; + } + + public int getTotal() { + return total; + } + + public int getSucceeded() { + return succeeded.get(); + } + + public int getFailed() { + return failed.get(); + } + + public int getCompleted() { + return succeeded.get() + failed.get(); + } + + @JsonProperty("isFinished") + public boolean isFinished() { + return getCompleted() >= total; + } +} diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/service/HttpRecordService.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/service/HttpRecordService.java new file mode 100644 index 000000000000..c824d356a5a7 --- /dev/null +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/service/HttpRecordService.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.service; + + +import org.apache.shenyu.admin.record.entity.RecordTaskInfo; +import org.apache.shenyu.admin.record.entity.ReplayProgress; + +import java.util.List; + +public interface HttpRecordService { + + /** + * List all available record task IDs from the storage directory. + * + * @return list of record task metadata + */ + List listRecordTasks(); + + /** + * Start an asynchronous replay of all records belonging to the given taskId. + * + * @param taskId the task id, maps to {storagePath}/{taskId}.jsonl + * @param targetHost base URL of the gateway to replay against + * @return a replayId that can be used to poll progress + */ + String startReplay(String taskId, String targetHost); + + /** + * Return the current progress of a replay job. + * + * @param replayId the id returned by {@link #startReplay} + * @return progress snapshot, or {@code null} if the id is unknown + */ + ReplayProgress getReplayProgress(String replayId); +} diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/service/impl/HttpRecordServiceImpl.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/service/impl/HttpRecordServiceImpl.java new file mode 100644 index 000000000000..a920cfa1ca4c --- /dev/null +++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/service/impl/HttpRecordServiceImpl.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.service.impl; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import okhttp3.Response; +import org.apache.commons.lang3.StringUtils; +import org.apache.shenyu.admin.config.properties.HttpRecordProperties; +import org.apache.shenyu.admin.model.dto.ShenyuHttpRequestRecordDTO; +import org.apache.shenyu.admin.record.HttpRecordRepository; +import org.apache.shenyu.admin.record.entity.RecordTaskInfo; +import org.apache.shenyu.admin.record.entity.ReplayProgress; +import org.apache.shenyu.admin.service.HttpRecordService; +import org.apache.shenyu.admin.utils.HttpUtils; +import org.apache.shenyu.common.constant.Constants; +import org.apache.shenyu.common.utils.UUIDUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +@Service +public class HttpRecordServiceImpl implements HttpRecordService { + + private static final Logger LOG = LoggerFactory.getLogger(HttpRecordServiceImpl.class); + + private final HttpUtils httpUtils; + + private final ThreadPoolExecutor replayExecutor; + + private final HttpRecordProperties properties; + + private final Cache progressMap = Caffeine.newBuilder() + .expireAfterWrite(1, TimeUnit.HOURS) + .build(); + + private final HttpRecordRepository httpRecordRepository; + + public HttpRecordServiceImpl(final ThreadPoolExecutor replayExecutor, + final HttpRecordProperties properties, final HttpUtils httpUtils, final Map repositoryMap) { + this.replayExecutor = replayExecutor; + this.properties = properties; + this.httpUtils = httpUtils; + this.httpRecordRepository = repositoryMap.getOrDefault(properties.getStorageType(), repositoryMap.get("local")); + } + + + @Override + public String startReplay(final String taskId, final String targetHost) { + List records = httpRecordRepository.loadRecords(taskId); + if (records.isEmpty()) { + throw new IllegalArgumentException("No records found for taskId: " + taskId); + } + + String replayId = UUIDUtils.getInstance().generateShortUuid(); + ReplayProgress progress = new ReplayProgress(replayId, taskId, records.size()); + progressMap.put(replayId, progress); + + for (ShenyuHttpRequestRecordDTO.Record record : records) { + replayExecutor.submit(() -> replayRecord(record, targetHost, progress)); + } + + LOG.info("Replay started. replayId={}, taskId={}, totalRecords={}, targetHost={}", + replayId, taskId, records.size(), targetHost); + return replayId; + } + + @Override + public ReplayProgress getReplayProgress(final String replayId) { + return progressMap.getIfPresent(replayId); + } + + + @Override + public List listRecordTasks() { + return httpRecordRepository.listRecordTasks(); + } + + private void replayRecord(final ShenyuHttpRequestRecordDTO.Record record, + final String targetHost, final ReplayProgress progress) { + String url = buildUrl(targetHost, record.getRequestUri(), record.getQueryParams()); + try { + doReplay(url, record); + progress.incrementSucceeded(); + } catch (Exception e) { + LOG.warn("Failed to replay record. method={}, url={}, traceId={}, error={}", + record.getMethod(), url, record.getTraceId(), e.getMessage()); + progress.incrementFailed(); + } + } + + private void doReplay(final String url, final ShenyuHttpRequestRecordDTO.Record record) throws IOException { + Map headers = buildReplayHeaders(record.getRequestHeaders()); + String body = StringUtils.defaultString(record.getRequestBody(), ""); + HttpUtils.HTTPMethod method = HttpUtils.HTTPMethod.fromValue(record.getMethod()); + try (Response response = httpUtils.requestJson(url, body, headers, method)) { + LOG.debug("Replayed {} {} -> status={}, traceId={}", + record.getMethod(), url, response.code(), record.getTraceId()); + } + } + + private String buildUrl(final String targetHost, final String requestUri, final String queryParams) { + String base = StringUtils.stripEnd(targetHost, "/") + requestUri; + if (StringUtils.isNotBlank(queryParams)) { + return base + "?" + queryParams; + } + return base; + } + + private Map buildReplayHeaders(final Map originalHeaders) { + Map headers = new HashMap<>(); + if (Objects.nonNull(originalHeaders)) { + for (Map.Entry entry : originalHeaders.entrySet()) { + String name = entry.getKey(); + if ("Content-Length".equalsIgnoreCase(name) || "Host".equalsIgnoreCase(name)) { + continue; + } + headers.put(name, entry.getValue()); + } + } + headers.put(Constants.X_SHENYU_REPLAY, "true"); + return headers; + } + +} diff --git a/shenyu-admin/src/main/resources/application.yml b/shenyu-admin/src/main/resources/application.yml index a0850d0bea18..6ba43bb5b597 100755 --- a/shenyu-admin/src/main/resources/application.yml +++ b/shenyu-admin/src/main/resources/application.yml @@ -227,12 +227,18 @@ shenyu: apiServer: "https://127.0.0.1:6443" token: "token" caCertPath: "/etc/kubernetes/pki/ca.crt" + + http-record: + storage-type: "local" + storage-path: "/tmp/shenyu-records" + config: import: max-entry-size: 104857600 max-total-size: 209715200 max-entry-count: 1000 + springdoc: api-docs: enabled: true diff --git a/shenyu-admin/src/main/resources/sql-script/h2/schema.sql b/shenyu-admin/src/main/resources/sql-script/h2/schema.sql index cfb561e4148b..5506a26f1f9a 100644 --- a/shenyu-admin/src/main/resources/sql-script/h2/schema.sql +++ b/shenyu-admin/src/main/resources/sql-script/h2/schema.sql @@ -1647,3 +1647,38 @@ INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('1953049887387303917', '1346358560427216896', '1844026099075534902', '2023-08-31 06:59:01.000', '2023-08-31 06:59:01.000'); INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('1953049887387303918', '1346358560427216896', '1844026099075534903', '2023-08-31 06:59:01.000', '2023-08-31 06:59:01.000'); INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('1953049887387303919', '1346358560427216896', '1844026099075534904', '2023-08-31 06:59:01.000', '2023-08-31 06:59:01.000'); + +-- httpRecord +INSERT IGNORE INTO `plugin` (`id`, `name`, `config`, `role`, `sort`, `enabled`) VALUES ('67', 'httpRecord', null, 'HttpProcess', 200, 1); + +INSERT IGNORE INTO plugin_handle (`id`, `plugin_id`,`field`,`label`,`data_type`,`type`,`sort`,`ext_obj`) VALUES ('2067162479970447360', '67', 'startTime', 'startTime', 1, 2, 0, '{"required":"1","defaultValue":"1780287614000","placeholder":"startTime","rule":""}'); +INSERT IGNORE INTO plugin_handle (`id`, `plugin_id`,`field`,`label`,`data_type`,`type`,`sort`,`ext_obj`) VALUES ('2067162479970447361', '67', 'endTime', 'endTime', 1, 2, 0, '{"required":"1","defaultValue":"1780287814000","placeholder":"endTime","rule":""}'); +INSERT IGNORE INTO plugin_handle (`id`, `plugin_id`,`field`,`label`,`data_type`,`type`,`sort`,`ext_obj`) VALUES ('2068305908214779904', '67', 'maxBodySize', 'maxBodySize', 1, 3, 0, '{"required":"0","defaultValue":"524288","rule":""}'); +INSERT IGNORE INTO plugin_handle (`id`, `plugin_id`,`field`,`label`,`data_type`,`type`,`sort`,`ext_obj`) VALUES ('2068306022568284160', '67', 'batchSize', 'batchSize', 1, 3, 1, '{"required":"0","defaultValue":"100","rule":""}'); +INSERT IGNORE INTO plugin_handle (`id`, `plugin_id`,`field`,`label`,`data_type`,`type`,`sort`,`ext_obj`) VALUES ('2068306133750894592', '67', 'batchIntervalMs', 'batchIntervalMs', 1, 3, 0, '{"required":"0","defaultValue":"5000","rule":""}'); + +INSERT IGNORE INTO `namespace_plugin_rel` (`id`,`namespace_id`,`plugin_id`, `config`, `sort`, `enabled`, `date_created`, `date_updated`) VALUES ('1801816010882822191','649330b6-c2d7-4edc-be8e-8a54df9eb385','67', NULL, 199, 1, '2022-05-25 18:02:53.000', '2026-05-25 09:27:25.639'); + +INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('2060675900525121540', '1346358560427216896', '2060959853814128640', '2023-08-31 06:59:01', '2023-08-31 06:59:01'); +INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('2060675900525121541', '1346358560427216896', '2060553097071169537', '2023-08-31 07:22:07', '2023-08-31 07:22:07'); +INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('2060675900525121542', '1346358560427216896', '2060553097071169538', '2023-08-31 07:14:26', '2023-08-31 07:14:26'); +INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('2060675900525121543', '1346358560427216896', '2060553097071169539', '2023-08-31 07:22:07', '2023-08-31 07:22:07'); +INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('2060675900525121544', '1346358560427216896', '2060553097071169541', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('2060675900525121545', '1346358560427216896', '2060553097071169542', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('2060675900525121546', '1346358560427216896', '2060553097071169543', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('2060675900525121547', '1346358560427216896', '2060553097071169544', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('2060675900525121548', '1346358560427216896', '2060553097071169545', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('2060675900525121549', '1346358560427216896', '2060553097071169546', '2023-08-31 07:18:37', '2023-08-31 07:18:37'); +INSERT IGNORE INTO `permission` (`id`, `object_id`, `resource_id`, `date_created`, `date_updated`) VALUES ('2060675900525121560', '1346358560427216896', '2060553097071169550', '2023-08-31 07:22:07', '2023-08-31 07:22:07'); + +INSERT IGNORE INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`) VALUES ('2060959853814128640', '1346775491550474240', 'httpRecord', 'httpRecord', '/plug/httpRecord', 'httpRecord', 1, 0, 'key', 0, 0, '', 1); +INSERT IGNORE INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`) VALUES ('2060553097071169537', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.ADD', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:add', 1); +INSERT IGNORE INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`) VALUES ('2060553097071169538', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.QUERY', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:query', 1); +INSERT IGNORE INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`) VALUES ('2060553097071169539', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.EDIT', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:edit', 1); +INSERT IGNORE INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`) VALUES ('2060553097071169541', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SELECTOR.DELETE', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordSelector:delete', 1); +INSERT IGNORE INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`) VALUES ('2060553097071169542', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.ADD', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:add', 1); +INSERT IGNORE INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`) VALUES ('2060553097071169543', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.QUERY', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:query', 1); +INSERT IGNORE INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`) VALUES ('2060553097071169544', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.EDIT', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:edit', 1); +INSERT IGNORE INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`) VALUES ('2060553097071169545', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.RULE.DELETE', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecordRule:delete', 1); +INSERT IGNORE INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`) VALUES ('2060553097071169546', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.SYNCHRONIZE', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecord:modify', 1); +INSERT IGNORE INTO `resource` (`id`, `parent_id`, `title`, `name`, `url`, `component`, `resource_type`, `sort`, `icon`, `is_leaf`, `is_route`, `perms`, `status`) VALUES ('2060553097071169550', '2060959853814128640', 'SHENYU.BUTTON.PLUGIN.LIST', '', '', '', 2, 0, '', 1, 0, 'plugin:httpRecord:list', 1); diff --git a/shenyu-admin/src/main/resources/static/index.html b/shenyu-admin/src/main/resources/static/index.html index bb2bb70954f5..3e983929e7db 100644 --- a/shenyu-admin/src/main/resources/static/index.html +++ b/shenyu-admin/src/main/resources/static/index.html @@ -24,11 +24,11 @@ Apache ShenYu Gateway - +
- + diff --git a/shenyu-admin/src/test/java/org/apache/shenyu/admin/controller/HttpRecordControllerTest.java b/shenyu-admin/src/test/java/org/apache/shenyu/admin/controller/HttpRecordControllerTest.java new file mode 100644 index 000000000000..da418ea7c882 --- /dev/null +++ b/shenyu-admin/src/test/java/org/apache/shenyu/admin/controller/HttpRecordControllerTest.java @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.controller; + +import org.apache.shenyu.admin.disruptor.ShenyuHttpRequestRecordDisruptorPublisher; +import org.apache.shenyu.admin.model.dto.ShenyuHttpRequestRecordDTO; +import org.apache.shenyu.admin.record.entity.RecordTaskInfo; +import org.apache.shenyu.admin.record.entity.ReplayProgress; +import org.apache.shenyu.admin.service.HttpRecordService; +import org.apache.shenyu.common.constant.Constants; +import org.apache.shenyu.common.exception.CommonErrorCode; +import org.apache.shenyu.common.utils.GsonUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.zip.CRC32; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.BDDMockito.given; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@ExtendWith(MockitoExtension.class) +class HttpRecordControllerTest { + + private MockMvc mockMvc; + + @Mock + private ShenyuHttpRequestRecordDisruptorPublisher publisher; + + @Mock + private HttpRecordService httpRecordService; + + @InjectMocks + private HttpRecordController httpRecordController; + + private List recordList; + + @BeforeEach + public void setUp() { + this.mockMvc = MockMvcBuilders.standaloneSetup(httpRecordController).build(); + ShenyuHttpRequestRecordDTO.Record record1 = new ShenyuHttpRequestRecordDTO.Record(); + record1.setMethod("GET"); + record1.setStatus(200); + record1.setQueryParams("test1=test1&test2=test2"); + record1.setResponseBody("testResponseBody"); + record1.setRequestUri("/testRequestUri"); + ShenyuHttpRequestRecordDTO.Record record2 = new ShenyuHttpRequestRecordDTO.Record(); + record2.setMethod("POST"); + record2.setStatus(200); + record2.setRequestBody("testRequestBody2"); + record2.setRequestUri("/testRequestUri2"); + record2.setResponseBody("testResponseBody2"); + this.recordList = Arrays.asList(record1, record2); + } + + @Test + public void testUploadHappyPath() throws Exception { + ArgumentCaptor captor = ArgumentCaptor.forClass(ShenyuHttpRequestRecordDTO.class); + byte[] content = GsonUtils.getInstance().toJson(recordList).getBytes(StandardCharsets.UTF_8); + CRC32 crc32 = new CRC32(); + crc32.update(content); + long crc32Value = crc32.getValue(); + mockMvc.perform(MockMvcRequestBuilders.post("/record/upload-record") + .header(Constants.X_RECORD_CRC32, crc32Value) + .contentType(MediaType.APPLICATION_JSON) + .content(content) + ).andExpect(status().isOk()).andReturn(); + Mockito.verify(publisher, Mockito.times(1)).publish(captor.capture()); + ShenyuHttpRequestRecordDTO dto = captor.getValue(); + assertEquals(2, dto.getRecords().size()); + assertEquals("GET", dto.getRecords().get(0).getMethod()); + assertEquals("/testRequestUri", dto.getRecords().get(0).getRequestUri()); + assertEquals("POST", dto.getRecords().get(1).getMethod()); + } + + @Test + public void testUploadNoCrc32() throws Exception { + mockMvc.perform(MockMvcRequestBuilders.post("/record/upload-record") + .contentType(MediaType.APPLICATION_JSON) + .content(GsonUtils.getInstance().toJson(recordList).getBytes(StandardCharsets.UTF_8)) + ).andExpect(status().is(400)).andReturn(); + Mockito.verify(publisher, Mockito.times(0)).publish(Mockito.any()); + } + + @Test + public void testUploadCrc32ValidationFailed() throws Exception { + byte[] content = GsonUtils.getInstance().toJson(recordList).getBytes(StandardCharsets.UTF_8); + long wrongCrc32 = 99999L; + + mockMvc.perform(MockMvcRequestBuilders.post("/record/upload-record") + .header(Constants.X_RECORD_CRC32, wrongCrc32) + .contentType(MediaType.APPLICATION_JSON) + .content(content) + ).andExpect(status().isOk()); + + Mockito.verify(publisher, Mockito.never()).publish(Mockito.any()); + } + + @Test + public void testUploadWithEmptyRecords() throws Exception { + byte[] content = "[]".getBytes(StandardCharsets.UTF_8); + CRC32 crc32 = new CRC32(); + crc32.update(content); + long crc32Value = crc32.getValue(); + + mockMvc.perform(MockMvcRequestBuilders.post("/record/upload-record") + .header(Constants.X_RECORD_CRC32, crc32Value) + .contentType(MediaType.APPLICATION_JSON) + .content(content) + ).andExpect(status().isOk()); + + Mockito.verify(publisher, Mockito.never()).publish(Mockito.any()); + } + + @Test + public void testUploadWithInvalidJsonBody() throws Exception { + byte[] content = "not json".getBytes(StandardCharsets.UTF_8); + CRC32 crc32 = new CRC32(); + crc32.update(content); + long crc32Value = crc32.getValue(); + + mockMvc.perform(MockMvcRequestBuilders.post("/record/upload-record") + .header(Constants.X_RECORD_CRC32, crc32Value) + .contentType(MediaType.APPLICATION_JSON) + .content(content) + ).andExpect(status().isOk()); + + Mockito.verify(publisher, Mockito.never()).publish(Mockito.any()); + } + + @Test + public void testListRecordTasksWithResults() throws Exception { + List tasks = Arrays.asList( + new RecordTaskInfo("task1", 10, 1000L), + new RecordTaskInfo("task2", 20, 2000L) + ); + given(httpRecordService.listRecordTasks()).willReturn(tasks); + + mockMvc.perform(MockMvcRequestBuilders.get("/record")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(CommonErrorCode.SUCCESSFUL)) + .andExpect(jsonPath("$.data").isArray()) + .andExpect(jsonPath("$.data.length()").value(2)) + .andExpect(jsonPath("$.data[0].taskId").value("task1")) + .andExpect(jsonPath("$.data[0].recordCount").value(10)) + .andExpect(jsonPath("$.data[1].taskId").value("task2")); + } + + @Test + public void testListRecordTasksEmpty() throws Exception { + given(httpRecordService.listRecordTasks()).willReturn(new ArrayList<>()); + + mockMvc.perform(MockMvcRequestBuilders.get("/record")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(CommonErrorCode.SUCCESSFUL)) + .andExpect(jsonPath("$.data").isArray()) + .andExpect(jsonPath("$.data.length()").value(0)); + } + + @Test + public void testStartReplaySuccess() throws Exception { + given(httpRecordService.startReplay("task1", "http://127.0.0.1:9195")).willReturn("replay123"); + + mockMvc.perform(MockMvcRequestBuilders.post("/record/replay") + .param("taskId", "task1") + .param("targetHost", "http://127.0.0.1:9195") + ).andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(CommonErrorCode.SUCCESSFUL)) + .andExpect(jsonPath("$.data").value("replay123")); + } + + @Test + public void testStartReplayWithBlankTaskId() throws Exception { + mockMvc.perform(MockMvcRequestBuilders.post("/record/replay") + .param("taskId", "") + .param("targetHost", "http://127.0.0.1:9195") + ).andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(CommonErrorCode.ERROR)) + .andExpect(jsonPath("$.message").value("taskId and targetHost must not be blank")); + } + + @Test + public void testStartReplayWithBlankTargetHost() throws Exception { + mockMvc.perform(MockMvcRequestBuilders.post("/record/replay") + .param("taskId", "task1") + .param("targetHost", "") + ).andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(CommonErrorCode.ERROR)) + .andExpect(jsonPath("$.message").value("taskId and targetHost must not be blank")); + } + + @Test + public void testStartReplayThrowsIllegalArgumentException() throws Exception { + given(httpRecordService.startReplay("badTask", "http://127.0.0.1:9195")) + .willThrow(new IllegalArgumentException("Task not found")); + + mockMvc.perform(MockMvcRequestBuilders.post("/record/replay") + .param("taskId", "badTask") + .param("targetHost", "http://127.0.0.1:9195") + ).andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(CommonErrorCode.ERROR)) + .andExpect(jsonPath("$.message").value("Task not found")); + } + + @Test + public void testStartReplayThrowsGenericException() throws Exception { + given(httpRecordService.startReplay("task1", "http://127.0.0.1:9195")) + .willThrow(new RuntimeException("Internal error")); + + mockMvc.perform(MockMvcRequestBuilders.post("/record/replay") + .param("taskId", "task1") + .param("targetHost", "http://127.0.0.1:9195") + ).andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(CommonErrorCode.ERROR)) + .andExpect(jsonPath("$.message").value("Failed to start replay: Internal error")); + } + + @Test + public void testGetReplayProgressSuccess() throws Exception { + ReplayProgress progress = new ReplayProgress("replay123", "task1", 10); + progress.incrementSucceeded(); + progress.incrementFailed(); + given(httpRecordService.getReplayProgress("replay123")).willReturn(progress); + + mockMvc.perform(MockMvcRequestBuilders.get("/record/progress/{replayId}", "replay123")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(CommonErrorCode.SUCCESSFUL)) + .andExpect(jsonPath("$.data.replayId").value("replay123")) + .andExpect(jsonPath("$.data.taskId").value("task1")) + .andExpect(jsonPath("$.data.total").value(10)) + .andExpect(jsonPath("$.data.succeeded").value(1)) + .andExpect(jsonPath("$.data.failed").value(1)); + } + + @Test + public void testGetReplayProgressNotFound() throws Exception { + given(httpRecordService.getReplayProgress("unknown")).willReturn(null); + + mockMvc.perform(MockMvcRequestBuilders.get("/record/progress/{replayId}", "unknown")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(CommonErrorCode.ERROR)) + .andExpect(jsonPath("$.message").value("Replay not found for replayId: unknown")); + } +} diff --git a/shenyu-admin/src/test/java/org/apache/shenyu/admin/record/LocalJsonFileHttpRecordRepositoryTest.java b/shenyu-admin/src/test/java/org/apache/shenyu/admin/record/LocalJsonFileHttpRecordRepositoryTest.java new file mode 100644 index 000000000000..e933eb2ef731 --- /dev/null +++ b/shenyu-admin/src/test/java/org/apache/shenyu/admin/record/LocalJsonFileHttpRecordRepositoryTest.java @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.record; + +import org.apache.shenyu.admin.config.properties.HttpRecordProperties; +import org.apache.shenyu.admin.model.dto.ShenyuHttpRequestRecordDTO; +import org.apache.shenyu.admin.record.entity.RecordTaskInfo; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +class LocalJsonFileHttpRecordRepositoryTest { + + @TempDir + private Path tempDir; + + @Mock + private HttpRecordProperties properties; + + private LocalJsonFileHttpRecordRepository repository; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + when(properties.getStoragePath()).thenReturn(tempDir.toString()); + repository = new LocalJsonFileHttpRecordRepository(properties); + } + + private ShenyuHttpRequestRecordDTO.Record buildRecord(final String taskId, final String traceId, + final String method, final String uri, + final String queryParams, final String body, + final Map headers) { + ShenyuHttpRequestRecordDTO.Record record = new ShenyuHttpRequestRecordDTO.Record(); + record.setTaskId(taskId); + record.setTraceId(traceId); + record.setMethod(method); + record.setRequestUri(uri); + record.setQueryParams(queryParams); + record.setRequestBody(body); + record.setRequestHeaders(headers); + return record; + } + + private ShenyuHttpRequestRecordDTO buildDTO(final List records) { + ShenyuHttpRequestRecordDTO dto = new ShenyuHttpRequestRecordDTO(); + dto.setRecords(records); + return dto; + } + + @Test + void testSaveCreatesFileAndWritesRecords() throws IOException { + ShenyuHttpRequestRecordDTO.Record record = buildRecord("task-1", "trace-1", "GET", "/api/test", null, null, null); + repository.save(buildDTO(Collections.singletonList(record))); + + Path file = tempDir.resolve("task-1.jsonl"); + assertTrue(Files.exists(file)); + List lines = Files.readAllLines(file); + assertEquals(1, lines.size()); + assertTrue(lines.get(0).contains("\"traceId\":\"trace-1\"")); + assertTrue(lines.get(0).contains("\"method\":\"GET\"")); + } + + @Test + void testSaveGroupsByTaskId() throws IOException { + ShenyuHttpRequestRecordDTO.Record r1 = buildRecord("task-a", "trace-1", "GET", "/a", null, null, null); + ShenyuHttpRequestRecordDTO.Record r2 = buildRecord("task-b", "trace-2", "POST", "/b", null, null, null); + ShenyuHttpRequestRecordDTO.Record r3 = buildRecord("task-a", "trace-3", "PUT", "/c", null, null, null); + repository.save(buildDTO(Arrays.asList(r1, r2, r3))); + + Path fileA = tempDir.resolve("task-a.jsonl"); + Path fileB = tempDir.resolve("task-b.jsonl"); + assertTrue(Files.exists(fileA)); + assertTrue(Files.exists(fileB)); + assertEquals(2, Files.readAllLines(fileA).size()); + assertEquals(1, Files.readAllLines(fileB).size()); + } + + @Test + void testSaveAppendsToExistingFile() throws IOException { + ShenyuHttpRequestRecordDTO.Record r1 = buildRecord("task-1", "trace-1", "GET", "/a", null, null, null); + repository.save(buildDTO(Collections.singletonList(r1))); + + ShenyuHttpRequestRecordDTO.Record r2 = buildRecord("task-1", "trace-2", "POST", "/b", null, null, null); + repository.save(buildDTO(Collections.singletonList(r2))); + + Path file = tempDir.resolve("task-1.jsonl"); + List lines = Files.readAllLines(file); + assertEquals(2, lines.size()); + assertTrue(lines.get(0).contains("trace-1")); + assertTrue(lines.get(1).contains("trace-2")); + } + + @Test + void testSaveCreatesDirectoryIfNotExists() { + Path nestedDir = tempDir.resolve("nested").resolve("sub"); + when(properties.getStoragePath()).thenReturn(nestedDir.toString()); + + LocalJsonFileHttpRecordRepository repo = new LocalJsonFileHttpRecordRepository(properties); + ShenyuHttpRequestRecordDTO.Record record = buildRecord("task-1", "trace-1", "GET", "/api/test", null, null, null); + repo.save(buildDTO(Collections.singletonList(record))); + + assertTrue(Files.exists(nestedDir.resolve("task-1.jsonl"))); + } + + @Test + void testSaveWithEmptyRecords() { + repository.save(buildDTO(Collections.emptyList())); + + try (var paths = Files.list(tempDir)) { + assertEquals(0, paths.count()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Test + void testListRecordTasksReturnsTaskInfo() throws IOException { + ShenyuHttpRequestRecordDTO.Record r1 = buildRecord("task-1", "trace-1", "GET", "/a", null, null, null); + ShenyuHttpRequestRecordDTO.Record r2 = buildRecord("task-1", "trace-2", "POST", "/b", null, null, null); + repository.save(buildDTO(Arrays.asList(r1, r2))); + + List tasks = repository.listRecordTasks(); + assertEquals(1, tasks.size()); + RecordTaskInfo info = tasks.get(0); + assertEquals("task-1", info.getTaskId()); + assertEquals(2, info.getRecordCount()); + assertTrue(info.getLastModifiedMs() > 0); + } + + @Test + void testListRecordTasksEmptyWhenDirNotExists() { + when(properties.getStoragePath()).thenReturn(tempDir.resolve("nonexistent").toString()); + LocalJsonFileHttpRecordRepository repo = new LocalJsonFileHttpRecordRepository(properties); + + List tasks = repo.listRecordTasks(); + assertTrue(tasks.isEmpty()); + } + + @Test + void testListRecordTasksIgnoresNonJsonlFiles() throws IOException { + Files.writeString(tempDir.resolve("task-1.jsonl"), "{\"taskId\":\"task-1\"}" + System.lineSeparator()); + Files.writeString(tempDir.resolve("readme.txt"), "not a record file"); + Files.writeString(tempDir.resolve("config.yaml"), "key: value"); + + List tasks = repository.listRecordTasks(); + assertEquals(1, tasks.size()); + assertEquals("task-1", tasks.get(0).getTaskId()); + } + + @Test + void testListRecordTasksEmptyDir() { + List tasks = repository.listRecordTasks(); + assertTrue(tasks.isEmpty()); + } + + @Test + void testLoadRecordsReturnsCorrectRecords() throws IOException { + Map headers = new HashMap<>(); + headers.put("Content-Type", "application/json"); + ShenyuHttpRequestRecordDTO.Record record = buildRecord("task-1", "trace-1", "POST", "/api/test", + "foo=bar", "{\"key\":\"val\"}", headers); + repository.save(buildDTO(Collections.singletonList(record))); + + List loaded = repository.loadRecords("task-1"); + assertEquals(1, loaded.size()); + ShenyuHttpRequestRecordDTO.Record r = loaded.get(0); + assertEquals("task-1", r.getTaskId()); + assertEquals("trace-1", r.getTraceId()); + assertEquals("POST", r.getMethod()); + assertEquals("/api/test", r.getRequestUri()); + assertEquals("foo=bar", r.getQueryParams()); + assertEquals("{\"key\":\"val\"}", r.getRequestBody()); + assertNotNull(r.getRequestHeaders()); + assertEquals("application/json", r.getRequestHeaders().get("Content-Type")); + } + + @Test + void testLoadRecordsReturnsEmptyWhenFileNotExists() { + List loaded = repository.loadRecords("nonexistent"); + assertTrue(loaded.isEmpty()); + } + + @Test + void testLoadRecordsSkipsBlankLines() throws IOException { + Path file = tempDir.resolve("task-1.jsonl"); + ShenyuHttpRequestRecordDTO.Record record = buildRecord("task-1", "trace-1", "GET", "/a", null, null, null); + String jsonLine = new com.google.gson.Gson().toJson(record); + Files.writeString(file, jsonLine + System.lineSeparator() + + System.lineSeparator() + + " " + System.lineSeparator() + + jsonLine + System.lineSeparator()); + + List loaded = repository.loadRecords("task-1"); + assertEquals(2, loaded.size()); + } + + @Test + void testLoadRecordsThrowsOnInvalidJson() throws IOException { + Path file = tempDir.resolve("task-1.jsonl"); + ShenyuHttpRequestRecordDTO.Record record = buildRecord("task-1", "trace-1", "GET", "/a", null, null, null); + String jsonLine = new com.google.gson.Gson().toJson(record); + Files.writeString(file, jsonLine + System.lineSeparator() + + "this is not json" + System.lineSeparator()); + + assertThrows(com.google.gson.JsonSyntaxException.class, () -> repository.loadRecords("task-1")); + } + + @Test + void testSaveAndLoadRoundTrip() { + Map headers = new HashMap<>(); + headers.put("Authorization", "Bearer token123"); + headers.put("X-Custom", "value"); + ShenyuHttpRequestRecordDTO.Record record = buildRecord("task-1", "trace-abc", "POST", "/api/submit", + "page=1&size=10", "{\"name\":\"test\"}", headers); + repository.save(buildDTO(Collections.singletonList(record))); + + List loaded = repository.loadRecords("task-1"); + assertEquals(1, loaded.size()); + ShenyuHttpRequestRecordDTO.Record r = loaded.get(0); + assertEquals("task-1", r.getTaskId()); + assertEquals("trace-abc", r.getTraceId()); + assertEquals("POST", r.getMethod()); + assertEquals("/api/submit", r.getRequestUri()); + assertEquals("page=1&size=10", r.getQueryParams()); + assertEquals("{\"name\":\"test\"}", r.getRequestBody()); + assertEquals("Bearer token123", r.getRequestHeaders().get("Authorization")); + assertEquals("value", r.getRequestHeaders().get("X-Custom")); + } + + @Test + void testListRecordTasksWithMultipleFiles() throws IOException { + ShenyuHttpRequestRecordDTO.Record r1 = buildRecord("alpha", "t1", "GET", "/a", null, null, null); + ShenyuHttpRequestRecordDTO.Record r2 = buildRecord("beta", "t2", "POST", "/b", null, null, null); + ShenyuHttpRequestRecordDTO.Record r3 = buildRecord("beta", "t3", "PUT", "/c", null, null, null); + repository.save(buildDTO(Arrays.asList(r1, r2, r3))); + + List tasks = repository.listRecordTasks(); + assertEquals(2, tasks.size()); + Map taskMap = new HashMap<>(); + for (RecordTaskInfo info : tasks) { + taskMap.put(info.getTaskId(), info); + } + assertTrue(taskMap.containsKey("alpha")); + assertTrue(taskMap.containsKey("beta")); + assertEquals(1, taskMap.get("alpha").getRecordCount()); + assertEquals(2, taskMap.get("beta").getRecordCount()); + } + + @Test + void testLoadRecordsWithInvalidTaskIdPathTraversal() { + List loaded = repository.loadRecords("../../etc/passwd"); + assertTrue(loaded.isEmpty()); + } + + @Test + void testLoadRecordsWithInvalidTaskIdNull() { + List loaded = repository.loadRecords(null); + assertTrue(loaded.isEmpty()); + } + + @Test + void testLoadRecordsWithInvalidTaskIdBlank() { + List loaded = repository.loadRecords(""); + assertTrue(loaded.isEmpty()); + loaded = repository.loadRecords(" "); + assertTrue(loaded.isEmpty()); + } + + @Test + void testLoadRecordsWithInvalidTaskIdSpecialChars() { + assertTrue(repository.loadRecords("task/1").isEmpty()); + assertTrue(repository.loadRecords("task\\1").isEmpty()); + assertTrue(repository.loadRecords("task.1").isEmpty()); + assertTrue(repository.loadRecords("task id").isEmpty()); + assertTrue(repository.loadRecords("task!@#").isEmpty()); + } + + @Test + void testLoadRecordsWithValidTaskIdCharacters() throws IOException { + ShenyuHttpRequestRecordDTO.Record record = buildRecord("task-1_abc", "trace-1", "GET", "/a", null, null, null); + repository.save(buildDTO(Collections.singletonList(record))); + + List loaded = repository.loadRecords("task-1_abc"); + assertEquals(1, loaded.size()); + assertEquals("task-1_abc", loaded.get(0).getTaskId()); + } + + @Test + void testLoadRecordsWithMultipleRecords() { + ShenyuHttpRequestRecordDTO.Record r1 = buildRecord("task-1", "trace-1", "GET", "/a", null, null, null); + ShenyuHttpRequestRecordDTO.Record r2 = buildRecord("task-1", "trace-2", "POST", "/b", "q=1", null, null); + ShenyuHttpRequestRecordDTO.Record r3 = buildRecord("task-1", "trace-3", "PUT", "/c", null, "{}", null); + repository.save(buildDTO(Arrays.asList(r1, r2, r3))); + + List loaded = repository.loadRecords("task-1"); + assertEquals(3, loaded.size()); + assertEquals("GET", loaded.get(0).getMethod()); + assertEquals("POST", loaded.get(1).getMethod()); + assertEquals("PUT", loaded.get(2).getMethod()); + assertEquals("q=1", loaded.get(1).getQueryParams()); + assertEquals("{}", loaded.get(2).getRequestBody()); + } +} diff --git a/shenyu-admin/src/test/java/org/apache/shenyu/admin/service/impl/HttpRecordServiceImplTest.java b/shenyu-admin/src/test/java/org/apache/shenyu/admin/service/impl/HttpRecordServiceImplTest.java new file mode 100644 index 000000000000..0697c996656a --- /dev/null +++ b/shenyu-admin/src/test/java/org/apache/shenyu/admin/service/impl/HttpRecordServiceImplTest.java @@ -0,0 +1,383 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.admin.service.impl; + +import okhttp3.Response; +import org.apache.shenyu.admin.config.properties.HttpRecordProperties; +import org.apache.shenyu.admin.model.dto.ShenyuHttpRequestRecordDTO; +import org.apache.shenyu.admin.record.HttpRecordRepository; +import org.apache.shenyu.admin.record.entity.RecordTaskInfo; +import org.apache.shenyu.admin.record.entity.ReplayProgress; +import org.apache.shenyu.admin.utils.HttpUtils; +import org.apache.shenyu.common.constant.Constants; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadPoolExecutor; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class HttpRecordServiceImplTest { + + @Mock + private ThreadPoolExecutor replayExecutor; + + @Mock + private HttpRecordProperties properties; + + @Mock + private HttpUtils httpUtils; + + @Mock + private HttpRecordRepository localRepository; + + @Mock + private HttpRecordRepository hdfsRepository; + + private HttpRecordServiceImpl service; + + @BeforeEach + void setUp() throws IOException { + MockitoAnnotations.openMocks(this); + + when(properties.getStorageType()).thenReturn("local"); + Map repositoryMap = new HashMap<>(); + repositoryMap.put("local", localRepository); + repositoryMap.put("hdfs", hdfsRepository); + + doAnswer(invocation -> { + ((Runnable) invocation.getArgument(0)).run(); + return null; + }).when(replayExecutor).submit(any(Runnable.class)); + + Response mockResponse = mock(Response.class); + when(mockResponse.code()).thenReturn(200); + when(httpUtils.requestJson(anyString(), anyString(), anyMap(), any(HttpUtils.HTTPMethod.class))) + .thenReturn(mockResponse); + + service = new HttpRecordServiceImpl(replayExecutor, properties, httpUtils, repositoryMap); + } + + private ShenyuHttpRequestRecordDTO.Record buildRecord(final String method, final String uri, + final String queryParams, final String body, + final Map headers) { + ShenyuHttpRequestRecordDTO.Record record = new ShenyuHttpRequestRecordDTO.Record(); + record.setTaskId("task-1"); + record.setTraceId("trace-1"); + record.setMethod(method); + record.setRequestUri(uri); + record.setQueryParams(queryParams); + record.setRequestBody(body); + record.setRequestHeaders(headers); + return record; + } + + @Test + void testListRecordTasks() { + RecordTaskInfo taskInfo = new RecordTaskInfo("task-1", 10L, System.currentTimeMillis()); + when(localRepository.listRecordTasks()).thenReturn(Collections.singletonList(taskInfo)); + + List result = service.listRecordTasks(); + + assertEquals(1, result.size()); + assertEquals("task-1", result.get(0).getTaskId()); + verify(localRepository).listRecordTasks(); + } + + @Test + void testStartReplayThrowsWhenNoRecords() { + when(localRepository.loadRecords("empty-task")).thenReturn(Collections.emptyList()); + + assertThrows(IllegalArgumentException.class, () -> service.startReplay("empty-task", "http://localhost")); + } + + @Test + void testStartReplayReturnsReplayId() { + ShenyuHttpRequestRecordDTO.Record record = buildRecord("GET", "/api/test", null, null, null); + when(localRepository.loadRecords("task-1")).thenReturn(Collections.singletonList(record)); + + String replayId = service.startReplay("task-1", "http://localhost:9195"); + + assertNotNull(replayId); + } + + @Test + void testStartReplayProgressTotalMatchesRecordCount() { + List records = Arrays.asList( + buildRecord("GET", "/api/a", null, null, null), + buildRecord("POST", "/api/b", null, "{}", null), + buildRecord("PUT", "/api/c", null, "{}", null) + ); + when(localRepository.loadRecords("task-1")).thenReturn(records); + + String replayId = service.startReplay("task-1", "http://localhost:9195"); + + ReplayProgress progress = service.getReplayProgress(replayId); + assertNotNull(progress); + assertEquals(3, progress.getTotal()); + assertEquals(3, progress.getSucceeded()); + assertEquals(0, progress.getFailed()); + } + + @Test + void testGetReplayProgressReturnsNullForUnknownId() { + assertNull(service.getReplayProgress("nonexistent")); + } + + @Test + void testReplaySuccessIncrementsSucceeded() { + ShenyuHttpRequestRecordDTO.Record record = buildRecord("GET", "/api/test", null, null, null); + when(localRepository.loadRecords("task-1")).thenReturn(Collections.singletonList(record)); + + String replayId = service.startReplay("task-1", "http://localhost:9195"); + + ReplayProgress progress = service.getReplayProgress(replayId); + assertNotNull(progress); + assertEquals(1, progress.getSucceeded()); + assertEquals(0, progress.getFailed()); + assertTrue(progress.isFinished()); + } + + @Test + void testReplayFailureIncrementsFailed() throws IOException { + ShenyuHttpRequestRecordDTO.Record record = buildRecord("GET", "/api/test", null, null, null); + when(localRepository.loadRecords("task-1")).thenReturn(Collections.singletonList(record)); + when(httpUtils.requestJson(anyString(), anyString(), anyMap(), any(HttpUtils.HTTPMethod.class))) + .thenThrow(new IOException("connection refused")); + + String replayId = service.startReplay("task-1", "http://localhost:9195"); + + ReplayProgress progress = service.getReplayProgress(replayId); + assertNotNull(progress); + assertEquals(0, progress.getSucceeded()); + assertEquals(1, progress.getFailed()); + assertTrue(progress.isFinished()); + } + + @Test + void testReplayBuildsUrlWithQueryParams() throws IOException { + ShenyuHttpRequestRecordDTO.Record record = buildRecord("GET", "/api/test", "foo=bar&baz=1", null, null); + when(localRepository.loadRecords("task-1")).thenReturn(Collections.singletonList(record)); + + service.startReplay("task-1", "http://localhost:9195"); + + verify(httpUtils).requestJson( + eq("http://localhost:9195/api/test?foo=bar&baz=1"), + anyString(), + anyMap(), + any(HttpUtils.HTTPMethod.class) + ); + } + + @Test + void testReplayBuildsUrlWithoutQueryParams() throws IOException { + ShenyuHttpRequestRecordDTO.Record record = buildRecord("GET", "/api/test", null, null, null); + when(localRepository.loadRecords("task-1")).thenReturn(Collections.singletonList(record)); + + service.startReplay("task-1", "http://localhost:9195"); + + verify(httpUtils).requestJson( + eq("http://localhost:9195/api/test"), + anyString(), + anyMap(), + any(HttpUtils.HTTPMethod.class) + ); + } + + @Test + void testReplayStripsTrailingSlashFromHost() throws IOException { + ShenyuHttpRequestRecordDTO.Record record = buildRecord("GET", "/api/test", null, null, null); + when(localRepository.loadRecords("task-1")).thenReturn(Collections.singletonList(record)); + + service.startReplay("task-1", "http://localhost:9195/"); + + verify(httpUtils).requestJson( + eq("http://localhost:9195/api/test"), + anyString(), + anyMap(), + any(HttpUtils.HTTPMethod.class) + ); + } + + @SuppressWarnings("unchecked") + @Test + void testReplayHeadersFilterContentLengthAndHost() throws IOException { + Map originalHeaders = new HashMap<>(); + originalHeaders.put("Content-Type", "application/json"); + originalHeaders.put("Content-Length", "256"); + originalHeaders.put("Host", "original-host.com"); + originalHeaders.put("Authorization", "Bearer token"); + + ShenyuHttpRequestRecordDTO.Record record = buildRecord("POST", "/api/test", null, "{}", originalHeaders); + when(localRepository.loadRecords("task-1")).thenReturn(Collections.singletonList(record)); + + service.startReplay("task-1", "http://localhost:9195"); + + ArgumentCaptor> headersCaptor = ArgumentCaptor.forClass(Map.class); + verify(httpUtils).requestJson(anyString(), anyString(), headersCaptor.capture(), any(HttpUtils.HTTPMethod.class)); + + Map replayHeaders = headersCaptor.getValue(); + assertEquals("application/json", replayHeaders.get("Content-Type")); + assertEquals("Bearer token", replayHeaders.get("Authorization")); + assertNull(replayHeaders.get("Content-Length")); + assertNull(replayHeaders.get("Host")); + } + + @SuppressWarnings("unchecked") + @Test + void testReplayHeadersAddShenyuReplayFlag() throws IOException { + ShenyuHttpRequestRecordDTO.Record record = buildRecord("GET", "/api/test", null, null, null); + when(localRepository.loadRecords("task-1")).thenReturn(Collections.singletonList(record)); + + service.startReplay("task-1", "http://localhost:9195"); + + ArgumentCaptor> headersCaptor = ArgumentCaptor.forClass(Map.class); + verify(httpUtils).requestJson(anyString(), anyString(), headersCaptor.capture(), any(HttpUtils.HTTPMethod.class)); + + Map replayHeaders = headersCaptor.getValue(); + assertEquals("true", replayHeaders.get(Constants.X_SHENYU_REPLAY)); + } + + @SuppressWarnings("unchecked") + @Test + void testReplayHeadersWithNullOriginalHeaders() throws IOException { + ShenyuHttpRequestRecordDTO.Record record = buildRecord("GET", "/api/test", null, null, null); + when(localRepository.loadRecords("task-1")).thenReturn(Collections.singletonList(record)); + + service.startReplay("task-1", "http://localhost:9195"); + + ArgumentCaptor> headersCaptor = ArgumentCaptor.forClass(Map.class); + verify(httpUtils).requestJson(anyString(), anyString(), headersCaptor.capture(), any(HttpUtils.HTTPMethod.class)); + + Map replayHeaders = headersCaptor.getValue(); + assertEquals(1, replayHeaders.size()); + assertEquals("true", replayHeaders.get(Constants.X_SHENYU_REPLAY)); + } + + @Test + void testRepositorySelectionByStorageType() { + when(properties.getStorageType()).thenReturn("hdfs"); + Map repositoryMap = new HashMap<>(); + repositoryMap.put("local", localRepository); + repositoryMap.put("hdfs", hdfsRepository); + + HttpRecordServiceImpl svc = new HttpRecordServiceImpl(replayExecutor, properties, httpUtils, repositoryMap); + + RecordTaskInfo taskInfo = new RecordTaskInfo("task-1", 5L, System.currentTimeMillis()); + when(hdfsRepository.listRecordTasks()).thenReturn(Collections.singletonList(taskInfo)); + + List result = svc.listRecordTasks(); + assertEquals(1, result.size()); + verify(hdfsRepository).listRecordTasks(); + verify(localRepository, org.mockito.Mockito.never()).listRecordTasks(); + } + + @Test + void testRepositoryFallsBackToLocal() { + when(properties.getStorageType()).thenReturn("unknown"); + Map repositoryMap = new HashMap<>(); + repositoryMap.put("local", localRepository); + repositoryMap.put("hdfs", hdfsRepository); + + HttpRecordServiceImpl svc = new HttpRecordServiceImpl(replayExecutor, properties, httpUtils, repositoryMap); + + RecordTaskInfo taskInfo = new RecordTaskInfo("task-1", 5L, System.currentTimeMillis()); + when(localRepository.listRecordTasks()).thenReturn(Collections.singletonList(taskInfo)); + + List result = svc.listRecordTasks(); + assertEquals(1, result.size()); + verify(localRepository).listRecordTasks(); + } + + @Test + void testReplaySendsCorrectMethodAndBody() throws IOException { + ShenyuHttpRequestRecordDTO.Record record = buildRecord("POST", "/api/test", null, "{\"key\":\"value\"}", null); + when(localRepository.loadRecords("task-1")).thenReturn(Collections.singletonList(record)); + + service.startReplay("task-1", "http://localhost:9195"); + + verify(httpUtils).requestJson( + eq("http://localhost:9195/api/test"), + eq("{\"key\":\"value\"}"), + anyMap(), + eq(HttpUtils.HTTPMethod.POST) + ); + } + + @Test + void testReplaySendsEmptyBodyWhenNull() throws IOException { + ShenyuHttpRequestRecordDTO.Record record = buildRecord("GET", "/api/test", null, null, null); + when(localRepository.loadRecords("task-1")).thenReturn(Collections.singletonList(record)); + + service.startReplay("task-1", "http://localhost:9195"); + + verify(httpUtils).requestJson( + anyString(), + eq(""), + anyMap(), + any(HttpUtils.HTTPMethod.class) + ); + } + + @Test + void testStartReplaySubmitsAllRecordsToExecutor() { + List records = Arrays.asList( + buildRecord("GET", "/api/a", null, null, null), + buildRecord("POST", "/api/b", null, null, null) + ); + when(localRepository.loadRecords("task-1")).thenReturn(records); + + service.startReplay("task-1", "http://localhost:9195"); + + verify(replayExecutor, org.mockito.Mockito.times(2)).submit(any(Runnable.class)); + } + + @Test + void testStartReplayProgressTracksTaskId() { + ShenyuHttpRequestRecordDTO.Record record = buildRecord("GET", "/api/test", null, null, null); + when(localRepository.loadRecords("my-task")).thenReturn(Collections.singletonList(record)); + + String replayId = service.startReplay("my-task", "http://localhost:9195"); + + ReplayProgress progress = service.getReplayProgress(replayId); + assertNotNull(progress); + assertEquals("my-task", progress.getTaskId()); + assertEquals(replayId, progress.getReplayId()); + } +} diff --git a/shenyu-bootstrap/pom.xml b/shenyu-bootstrap/pom.xml index 4a0b181b6b90..254c93d8c600 100644 --- a/shenyu-bootstrap/pom.xml +++ b/shenyu-bootstrap/pom.xml @@ -576,6 +576,12 @@ + + org.apache.shenyu + shenyu-spring-boot-starter-plugin-http-record + ${project.version} + + diff --git a/shenyu-bootstrap/src/main/resources/application.yml b/shenyu-bootstrap/src/main/resources/application.yml index 9272726f6982..35ea74704142 100644 --- a/shenyu-bootstrap/src/main/resources/application.yml +++ b/shenyu-bootstrap/src/main/resources/application.yml @@ -266,6 +266,12 @@ shenyu: props: username: admin password: 123456 + http-record: + enabled: true + serverLists: http://localhost:9095 + props: + username: admin + password: 123456 exclude: enabled: false paths: diff --git a/shenyu-common/src/main/java/org/apache/shenyu/common/constant/Constants.java b/shenyu-common/src/main/java/org/apache/shenyu/common/constant/Constants.java index 5295c8552b8c..c68440dc23db 100644 --- a/shenyu-common/src/main/java/org/apache/shenyu/common/constant/Constants.java +++ b/shenyu-common/src/main/java/org/apache/shenyu/common/constant/Constants.java @@ -1068,6 +1068,32 @@ public interface Constants { */ String GB = "GB"; + + /** + * The path that upload record to admin. + */ + String RECORD_UPLOAD_PATH = "/record/upload-record"; + + /** + * X-Record-CRC32. + */ + String X_RECORD_CRC32 = "X-Record-CRC32"; + + /** + * X-Record-Storage-Path. + */ + String X_RECORD_STORAGE_PATH = "X-Record-Storage-Path"; + + /** + * X-Record-Task-Id. + */ + String X_RECORD_TASK_ID = "X-Record-Task-Id"; + + /** + * X-Shenyu-Replay. + */ + String X_SHENYU_REPLAY = "X-Shenyu-Replay"; + /** * String q. */ diff --git a/shenyu-common/src/main/java/org/apache/shenyu/common/dto/convert/rule/HttpRecordHandle.java b/shenyu-common/src/main/java/org/apache/shenyu/common/dto/convert/rule/HttpRecordHandle.java new file mode 100644 index 000000000000..c612171ef99b --- /dev/null +++ b/shenyu-common/src/main/java/org/apache/shenyu/common/dto/convert/rule/HttpRecordHandle.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.common.dto.convert.rule; + +import java.util.concurrent.TimeUnit; + +public class HttpRecordHandle { + + private Long startTime; + + private Long endTime; + + private String taskId; + + public static HttpRecordHandle newDefaultInstance() { + HttpRecordHandle httpRecordHandle = new HttpRecordHandle(); + httpRecordHandle.startTime = System.currentTimeMillis(); + httpRecordHandle.endTime = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(1); + return httpRecordHandle; + } + + public void setTaskId(final String taskId) { + this.taskId = taskId; + } + + public String getTaskId() { + return taskId; + } + + public void setStartTime(final Long startTime) { + this.startTime = startTime; + } + + public Long getStartTime() { + return startTime; + } + + public void setEndTime(final Long endTime) { + this.endTime = endTime; + } + + public Long getEndTime() { + return endTime; + } + +} diff --git a/shenyu-common/src/main/java/org/apache/shenyu/common/enums/PluginEnum.java b/shenyu-common/src/main/java/org/apache/shenyu/common/enums/PluginEnum.java index 8501238fa405..8ace24f2ba6a 100644 --- a/shenyu-common/src/main/java/org/apache/shenyu/common/enums/PluginEnum.java +++ b/shenyu-common/src/main/java/org/apache/shenyu/common/enums/PluginEnum.java @@ -295,7 +295,12 @@ public enum PluginEnum { /** * Basic-auth plugin enum. */ - BASIC_AUTH(150, 0, "basicAuth"); + BASIC_AUTH(150, 0, "basicAuth"), + + /** + * Record plugin enum. + */ + HTTP_RECORD(215, 0, "httpRecord"); /** * When the application starts, the plugin is cached and we can obtained by name. diff --git a/shenyu-plugin/pom.xml b/shenyu-plugin/pom.xml index bdcc7061e39e..0719b3b6cd4b 100644 --- a/shenyu-plugin/pom.xml +++ b/shenyu-plugin/pom.xml @@ -51,6 +51,7 @@ shenyu-plugin-fault-tolerance shenyu-plugin-ai shenyu-plugin-mcp-server + shenyu-plugin-record diff --git a/shenyu-plugin/shenyu-plugin-record/pom.xml b/shenyu-plugin/shenyu-plugin-record/pom.xml new file mode 100644 index 000000000000..08fbb29ce56e --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/pom.xml @@ -0,0 +1,40 @@ + + + + + + org.apache.shenyu + shenyu-plugin + 2.7.2-SNAPSHOT + + 4.0.0 + + shenyu-plugin-record + + + + org.apache.shenyu + shenyu-plugin-base + ${project.version} + + + com.squareup.okhttp3 + okhttp + + + \ No newline at end of file diff --git a/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/HttpRecordPlugin.java b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/HttpRecordPlugin.java new file mode 100644 index 000000000000..5a712e776992 --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/HttpRecordPlugin.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record; + +import org.apache.shenyu.common.constant.Constants; +import org.apache.shenyu.common.dto.RuleData; +import org.apache.shenyu.common.dto.SelectorData; +import org.apache.shenyu.common.dto.convert.rule.HttpRecordHandle; +import org.apache.shenyu.common.enums.PluginEnum; +import org.apache.shenyu.common.utils.UUIDUtils; +import org.apache.shenyu.plugin.api.ShenyuPluginChain; +import org.apache.shenyu.plugin.base.AbstractShenyuPlugin; +import org.apache.shenyu.plugin.base.utils.CacheKeyUtils; +import org.apache.shenyu.plugin.record.body.RecordServerHttpRequest; +import org.apache.shenyu.plugin.record.body.RecordServerHttpResponse; +import org.apache.shenyu.plugin.record.collector.HttpRecordCollector; +import org.apache.shenyu.plugin.record.entity.ShenyuHttpRequestRecord; +import org.apache.shenyu.plugin.record.handler.HttpRecordPluginDataHandler; +import org.apache.shenyu.plugin.record.utils.RecordUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.server.reactive.ServerHttpRequest; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Mono; + +import java.util.Map; +import java.util.Objects; + +/** + * HttpRecordPlugin. + * + *

Records HTTP request and response data within a configured time window, + * for later replay and debugging purposes.

+ */ +public class HttpRecordPlugin extends AbstractShenyuPlugin { + + private static final Logger LOG = LoggerFactory.getLogger(HttpRecordPlugin.class); + + @Override + protected Mono doExecute(final ServerWebExchange exchange, final ShenyuPluginChain chain, + final SelectorData selector, final RuleData rule) { + HttpRecordHandle httpRecordHandle = HttpRecordPluginDataHandler.CACHED_HANDLE.get() + .obtainHandle(CacheKeyUtils.INST.getKey(rule)); + if (Objects.isNull(httpRecordHandle)) { + LOG.error("httpRecord rule handle is null, skip recording for rule: {}", rule.getId()); + return chain.execute(exchange); + } + long currentTime = System.currentTimeMillis(); + if (currentTime < httpRecordHandle.getStartTime() || currentTime > httpRecordHandle.getEndTime()) { + return chain.execute(exchange); + } + ServerHttpRequest request = exchange.getRequest(); + Map headers = RecordUtils.getHeaders(request.getHeaders()); + if (headers.containsKey(Constants.X_SHENYU_REPLAY)) { + return chain.execute(exchange); + } + ShenyuHttpRequestRecord record = new ShenyuHttpRequestRecord(); + record.setTaskId(httpRecordHandle.getTaskId()); + record.setTraceId(UUIDUtils.getInstance().generateShortUuid()); + record.setMethod(request.getMethod().name()); + record.setQueryParams(request.getURI().getQuery()); + record.setRequestUri(request.getURI().getPath()); + record.setRequestHeaders(headers); + + RecordServerHttpRequest recordRequest = + new RecordServerHttpRequest(request, record); + RecordServerHttpResponse recordResponse = + new RecordServerHttpResponse(exchange.getResponse(), record, + HttpRecordCollector.getInstance(), exchange); + ServerWebExchange mutatedExchange = exchange.mutate() + .request(recordRequest) + .response(recordResponse) + .build(); + + return chain.execute(mutatedExchange); + } + + @Override + public String named() { + return PluginEnum.HTTP_RECORD.getName(); + } + + @Override + public int getOrder() { + return PluginEnum.HTTP_RECORD.getCode(); + } +} diff --git a/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/body/BodyWriter.java b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/body/BodyWriter.java new file mode 100644 index 000000000000..0cf5906269e3 --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/body/BodyWriter.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.body; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * BodyWriter. + * + *

A thread-safe writer that accumulates byte buffers into a byte array, + * with an optional maximum size limit to prevent OOM in the gateway.

+ */ +public class BodyWriter { + + private static final Logger LOG = LoggerFactory.getLogger(BodyWriter.class); + + private final int maxSize; + + private final AtomicBoolean isClosed = new AtomicBoolean(false); + + private boolean sizeExceeded; + + private final ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + private final WritableByteChannel channel = Channels.newChannel(stream); + + /** + * Constructor with no size limit. + */ + public BodyWriter() { + this(Integer.MAX_VALUE); + } + + /** + * Constructor with max size limit. + * + * @param maxSize the maximum number of bytes to capture + */ + public BodyWriter(final int maxSize) { + this.maxSize = maxSize; + } + + /** + * Write a byte buffer. If the accumulated size exceeds maxSize, further writes are silently dropped. + * + * @param buffer the byte buffer to write + */ + public void write(final ByteBuffer buffer) { + if (!isClosed.get()) { + if (sizeExceeded || stream.size() >= maxSize) { + sizeExceeded = true; + return; + } + try { + channel.write(buffer); + if (stream.size() > maxSize) { + sizeExceeded = true; + } + } catch (IOException e) { + isClosed.compareAndSet(false, true); + LOG.error("write buffer Failed.", e); + } + } + } + + /** + * Is empty. + * + * @return true if no data has been written + */ + public boolean isEmpty() { + return stream.size() == 0; + } + + /** + * Get current size. + * + * @return the number of bytes written so far + */ + public int size() { + return stream.size(); + } + + /** + * Whether the max size limit has been exceeded. + * + * @return true if the body was truncated + */ + public boolean isSizeExceeded() { + return sizeExceeded; + } + + /** + * Output the captured data as a UTF-8 string and close the writer. + * Calling output() multiple times is safe; subsequent calls return empty string. + * + * @return the captured body as a string, or empty string if no data + */ + public String output() { + if (isEmpty()) { + return ""; + } + try { + isClosed.compareAndSet(false, true); + return new String(stream.toByteArray(), StandardCharsets.UTF_8); + } catch (Exception e) { + LOG.error("Write failed: ", e); + return "Write failed: " + e.getMessage(); + } finally { + try { + stream.close(); + } catch (IOException e) { + LOG.error("Close stream error: ", e); + } + try { + channel.close(); + } catch (IOException e) { + LOG.error("Close channel error: ", e); + } + } + } +} diff --git a/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/body/RecordServerHttpRequest.java b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/body/RecordServerHttpRequest.java new file mode 100644 index 000000000000..65b4d0a7549a --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/body/RecordServerHttpRequest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.body; + +import org.apache.shenyu.plugin.record.config.HttpRecordCollectConfig; +import org.apache.shenyu.plugin.record.entity.ShenyuHttpRequestRecord; +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.http.server.reactive.ServerHttpRequest; +import org.springframework.http.server.reactive.ServerHttpRequestDecorator; +import reactor.core.publisher.Flux; +import reactor.util.annotation.NonNull; + +/** + * RecordServerHttpRequest. + * + *

Decorator that captures request body for recording. + * Body capture is limited by maxBodySize to prevent OOM.

+ */ +public class RecordServerHttpRequest extends ServerHttpRequestDecorator { + + private static final String TRUNCATED_PREFIX = "[TRUNCATED] "; + + private final ShenyuHttpRequestRecord record; + + private final int maxBodySize; + + /** + * Constructor. + * + * @param delegate the original request + * @param record the record object to populate + */ + public RecordServerHttpRequest(final ServerHttpRequest delegate, final ShenyuHttpRequestRecord record) { + super(delegate); + this.record = record; + this.maxBodySize = HttpRecordCollectConfig.INSTANCE.getRecordConfig().getMaxBodySize(); + } + + @Override + @NonNull + public Flux getBody() { + BodyWriter writer = new BodyWriter(maxBodySize); + return super.getBody().doOnNext(dataBuffer -> { + try (DataBuffer.ByteBufferIterator bufferIterator = dataBuffer.readableByteBuffers()) { + bufferIterator.forEachRemaining(byteBuffer -> writer.write(byteBuffer.asReadOnlyBuffer())); + } + }).doFinally(signal -> { + if (writer.isSizeExceeded()) { + record.setRequestBody(TRUNCATED_PREFIX + "request body exceeded max size of " + maxBodySize + " bytes"); + } else { + record.setRequestBody(writer.output()); + } + }); + } +} diff --git a/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/body/RecordServerHttpResponse.java b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/body/RecordServerHttpResponse.java new file mode 100644 index 000000000000..b670b7b69129 --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/body/RecordServerHttpResponse.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.body; + +import org.apache.shenyu.plugin.base.utils.MediaTypeUtils; +import org.apache.shenyu.plugin.record.collector.HttpRecordCollector; +import org.apache.shenyu.plugin.record.config.HttpRecordCollectConfig; +import org.apache.shenyu.plugin.record.entity.ShenyuHttpRequestRecord; +import org.apache.shenyu.plugin.record.utils.RecordUtils; +import org.reactivestreams.Publisher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.http.MediaType; +import org.springframework.http.server.reactive.ServerHttpResponse; +import org.springframework.http.server.reactive.ServerHttpResponseDecorator; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.util.annotation.NonNull; + +import java.util.Objects; + +/** + * RecordServerHttpResponse. + * + *

Decorator that captures response status, headers, and body for recording. + * When the response body is binary type, only metadata is captured.

+ */ +public class RecordServerHttpResponse extends ServerHttpResponseDecorator { + + private static final Logger LOG = LoggerFactory.getLogger(RecordServerHttpResponse.class); + + private static final String TRUNCATED_PREFIX = "[TRUNCATED] "; + + private final ShenyuHttpRequestRecord record; + + private final ServerWebExchange exchange; + + private final HttpRecordCollector httpRecordCollector; + + private final int maxBodySize; + + /** + * Constructor. + * + * @param delegate the original response + * @param record the record object to populate + * @param httpRecordCollector the collector to submit the record to + * @param exchange the server web exchange + */ + public RecordServerHttpResponse(final ServerHttpResponse delegate, final ShenyuHttpRequestRecord record, + final HttpRecordCollector httpRecordCollector, + final ServerWebExchange exchange) { + super(delegate); + this.record = record; + this.httpRecordCollector = httpRecordCollector; + this.exchange = exchange; + this.maxBodySize = HttpRecordCollectConfig.INSTANCE.getRecordConfig().getMaxBodySize(); + } + + @Override + public Mono writeWith(final Publisher body) { + return super.writeWith(appendResponse(body)); + } + + @NonNull + private Flux appendResponse(final Publisher body) { + if (Objects.nonNull(getStatusCode())) { + record.setStatus(getStatusCode().value()); + } + record.setResponseHeaders(RecordUtils.getHeaders(getHeaders())); + final MediaType mediaType = exchange.getResponse().getHeaders().getContentType(); + if (MediaTypeUtils.isByteType(mediaType)) { + httpRecordCollector.collect(record); + return Flux.from(body); + } + BodyWriter writer = new BodyWriter(maxBodySize); + return Flux.from(body).doOnNext(buffer -> { + if (RecordUtils.isNotBinaryType(getHeaders())) { + try (DataBuffer.ByteBufferIterator bufferIterator = buffer.readableByteBuffers()) { + bufferIterator.forEachRemaining(byteBuffer -> writer.write(byteBuffer.asReadOnlyBuffer())); + } + } + }).doFinally(signal -> { + if (writer.isSizeExceeded()) { + record.setResponseBody(TRUNCATED_PREFIX + "response body exceeded max size of " + maxBodySize + " bytes"); + } else { + record.setResponseBody(writer.output()); + } + httpRecordCollector.collect(record); + }); + } +} diff --git a/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/collector/HttpRecordCollector.java b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/collector/HttpRecordCollector.java new file mode 100644 index 000000000000..8dd60fe297d4 --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/collector/HttpRecordCollector.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.collector; + +import com.github.benmanes.caffeine.cache.CacheLoader; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import okhttp3.Headers; +import org.apache.commons.lang3.StringUtils; +import org.apache.shenyu.common.concurrent.MemorySafeTaskQueue; +import org.apache.shenyu.common.concurrent.ShenyuThreadFactory; +import org.apache.shenyu.common.concurrent.ShenyuThreadPoolExecutor; +import org.apache.shenyu.common.config.ShenyuConfig; +import org.apache.shenyu.common.constant.Constants; +import org.apache.shenyu.common.exception.CommonErrorCode; +import org.apache.shenyu.common.utils.GsonUtils; +import org.apache.shenyu.common.utils.Singleton; +import org.apache.shenyu.common.utils.ThreadUtils; +import org.apache.shenyu.plugin.api.utils.SpringBeanUtils; +import org.apache.shenyu.plugin.record.config.HttpRecordCollectConfig; +import org.apache.shenyu.plugin.record.config.ShenyuHttpRecordCenterConfig; +import org.apache.shenyu.plugin.record.entity.ShenyuHttpRequestRecord; +import org.apache.shenyu.plugin.record.utils.OkHttpTools; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.CRC32; + +/** + * HttpRecordCollector. + * + *

Collects recorded HTTP request/response data from the gateway plugin, + * buffers them in a lock-free queue, and uploads batches to the admin server.

+ */ +public class HttpRecordCollector { + + public static final HttpRecordCollector INSTANCE = new HttpRecordCollector(); + + private static final Logger LOG = LoggerFactory.getLogger(HttpRecordCollector.class); + + private static final long ACCESS_TOKEN_CACHE_TTL_HOURS = 24L; + + private static final int CONSUMER_POLL_INTERVAL_MS = 50; + + private static final int CONSUMER_ERROR_BACKOFF_MS = 100; + + private static final int SHUTDOWN_AWAIT_TIMEOUT_SECONDS = 30; + + private final AtomicBoolean started = new AtomicBoolean(false); + + private int maxBufferSize; + + private final AtomicInteger bufferSize = new AtomicInteger(0); + + private ConcurrentLinkedQueue bufferQueue; + + private ShenyuHttpRecordCenterConfig shenyuHttpRecordCenterConfig; + + private LoadingCache accessToken; + + private HttpRecordCollectConfig.RecordConfig config; + + private String username; + + private String password; + + private List serverList; + + private ShenyuThreadPoolExecutor threadExecutor; + + /** + * Start the collector: initialize config, login cache, buffer queue and consumer thread. + */ + public void start() { + if (!started.compareAndSet(false, true)) { + return; + } + shenyuHttpRecordCenterConfig = SpringBeanUtils.getInstance().getBean(ShenyuHttpRecordCenterConfig.class); + this.username = shenyuHttpRecordCenterConfig.getProps().getProperty(Constants.USER_NAME); + this.password = shenyuHttpRecordCenterConfig.getProps().getProperty(Constants.PASS_WORD); + this.serverList = Lists.newArrayList(Splitter.on(",").split(shenyuHttpRecordCenterConfig.getServerLists())); + config = HttpRecordCollectConfig.INSTANCE.getRecordConfig(); + this.accessToken = Caffeine.newBuilder() + .expireAfterWrite(ACCESS_TOKEN_CACHE_TTL_HOURS, TimeUnit.HOURS) + .build(new CacheLoader<>() { + @Override + public @Nullable String load(@NonNull final String server) { + try { + Optional login = doLogin(username, password, server.concat(Constants.LOGIN_PATH)); + return login.map(String::valueOf).orElse(null); + } catch (Exception e) { + LOG.error("Login admin url :{} is fail, will retry. cause: {} ", server, e.getMessage()); + return null; + } + } + }); + maxBufferSize = Math.max(config.getBatchSize() * 500, 5000); + bufferQueue = new ConcurrentLinkedQueue<>(); + ShenyuConfig shenyuConfig = Optional.ofNullable(Singleton.INST.get(ShenyuConfig.class)).orElse(new ShenyuConfig()); + final ShenyuConfig.SharedPool sharedPool = shenyuConfig.getSharedPool(); + threadExecutor = new ShenyuThreadPoolExecutor(sharedPool.getCorePoolSize(), + sharedPool.getMaximumPoolSize(), sharedPool.getKeepAliveTime(), TimeUnit.MILLISECONDS, + new MemorySafeTaskQueue<>(Constants.THE_256_MB), + ShenyuThreadFactory.create(shenyuConfig.getSharedPool().getPrefix(), true), + new ThreadPoolExecutor.AbortPolicy()); + threadExecutor.execute(this::consume); + } + + /** + * Stop the collector: signal the consumer thread to stop and shutdown the thread pool. + */ + public void stop() { + if (!started.compareAndSet(true, false)) { + return; + } + if (Objects.nonNull(threadExecutor)) { + threadExecutor.shutdown(); + try { + if (!threadExecutor.awaitTermination(SHUTDOWN_AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + LOG.warn("Thread executor did not terminate within {} seconds", SHUTDOWN_AWAIT_TIMEOUT_SECONDS); + threadExecutor.shutdownNow(); + } + } catch (InterruptedException e) { + threadExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + if (Objects.nonNull(accessToken)) { + accessToken.cleanUp(); + } + } + + /** + * Collect a record into the buffer queue. This method is lock-free and safe to call from + * the WebFlux Event Loop thread. + * + * @param record the http request record to collect + */ + public void collect(final ShenyuHttpRequestRecord record) { + if (Objects.isNull(record) || Objects.isNull(bufferQueue)) { + return; + } + if (bufferSize.get() < maxBufferSize) { + bufferSize.incrementAndGet(); + bufferQueue.offer(record); + } else { + LOG.warn("Record buffer is full, dropping record. traceId={}", record.getTraceId()); + } + } + + private void consume() { + int batchSize = config.getBatchSize(); + long batchIntervalMs = config.getBatchIntervalMs(); + List batch = new ArrayList<>(); + long lastFlushTime = System.currentTimeMillis(); + while (started.get()) { + try { + ShenyuHttpRequestRecord record = bufferQueue.poll(); + if (Objects.nonNull(record)) { + bufferSize.decrementAndGet(); + batch.add(record); + } + boolean batchFull = batch.size() >= batchSize; + boolean timeout = !batch.isEmpty() && System.currentTimeMillis() - lastFlushTime >= batchIntervalMs; + if (batchFull || timeout) { + flushBatch(batch); + batch.clear(); + lastFlushTime = System.currentTimeMillis(); + } else if (bufferQueue.isEmpty()) { + ThreadUtils.sleep(TimeUnit.MILLISECONDS, CONSUMER_POLL_INTERVAL_MS); + } + } catch (Throwable t) { + LOG.error("ShenyuHttpRequestRecord collect log error", t); + ThreadUtils.sleep(TimeUnit.MILLISECONDS, CONSUMER_ERROR_BACKOFF_MS); + } + } + if (!batch.isEmpty()) { + try { + flushBatch(batch); + } catch (Exception e) { + LOG.error("Failed to flush remaining records on shutdown", e); + } + } + } + + private void flushBatch(final List records) throws IOException { + String jsonPayload = GsonUtils.getInstance().toJson(records); + byte[] payloadBytes = jsonPayload.getBytes(StandardCharsets.UTF_8); + CRC32 crc32 = new CRC32(); + crc32.update(payloadBytes); + long checkSum = crc32.getValue(); + doUploadRecord(records, checkSum); + } + + private void doUploadRecord(final List records, final long checkSum) throws IOException { + for (String server : serverList) { + String contact = server.concat(Constants.RECORD_UPLOAD_PATH); + String token = this.accessToken.get(server); + if (StringUtils.isBlank(token)) { + throw new IllegalStateException("accessToken is blank, login may have failed for server: " + server); + } + Headers headers = new Headers.Builder().add(Constants.X_ACCESS_TOKEN, token) + .add(Constants.X_RECORD_CRC32, String.valueOf(checkSum)) + .build(); + OkHttpTools.getInstance().post(contact, GsonUtils.getInstance().toJson(records), headers); + } + } + + /** + * Gets instance. + * + * @return the singleton instance + */ + public static HttpRecordCollector getInstance() { + return INSTANCE; + } + + private static Optional doLogin(final String username, final String password, + final String url) throws IOException { + Map loginMap = new HashMap<>(2); + loginMap.put(Constants.LOGIN_NAME, username); + loginMap.put(Constants.PASS_WORD, password); + String result = OkHttpTools.getInstance().get(url, loginMap); + Map resultMap = GsonUtils.getInstance().convertToMap(result); + if (!String.valueOf(CommonErrorCode.SUCCESSFUL).equals(String.valueOf(resultMap.get(Constants.ADMIN_RESULT_CODE)))) { + return Optional.empty(); + } + String tokenJson = GsonUtils.getInstance().toJson(resultMap.get(Constants.ADMIN_RESULT_DATA)); + LOG.info("login success: {} ", tokenJson); + Map tokenMap = GsonUtils.getInstance().convertToMap(tokenJson); + return Optional.ofNullable(tokenMap.get(Constants.ADMIN_RESULT_TOKEN)); + } + +} diff --git a/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/config/HttpRecordCollectConfig.java b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/config/HttpRecordCollectConfig.java new file mode 100644 index 000000000000..c02729f5604c --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/config/HttpRecordCollectConfig.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.config; + +import java.util.Objects; +import java.util.Optional; + +/** + * Configuration holder for the http record plugin. + * + *

Stores the shared record config that is pushed from the admin server + * and read by both the gateway plugin and the collector thread.

+ */ +public class HttpRecordCollectConfig { + + public static final HttpRecordCollectConfig INSTANCE = new HttpRecordCollectConfig(); + + private static final int DEFAULT_MAX_BODY_SIZE = 524288; + + private static final int DEFAULT_BATCH_SIZE = 100; + + private static final long DEFAULT_BATCH_INTERVAL_MS = 5000L; + + private volatile RecordConfig recordConfig; + + /** + * Get record config. + * + * @return the record config, defaults to a new instance if not set + */ + public RecordConfig getRecordConfig() { + return Optional.ofNullable(recordConfig).orElse(new RecordConfig()); + } + + /** + * Set record config. + * + * @param recordConfig the record config to set + */ + public void setRecordConfig(final RecordConfig recordConfig) { + this.recordConfig = recordConfig; + } + + /** + * Record config properties. + */ + public static class RecordConfig { + + private Integer maxBodySize = DEFAULT_MAX_BODY_SIZE; + + private int batchSize = DEFAULT_BATCH_SIZE; + + private long batchIntervalMs = DEFAULT_BATCH_INTERVAL_MS; + + /** + * Get max body size in bytes. + * + * @return the max body size + */ + public Integer getMaxBodySize() { + return maxBodySize; + } + + /** + * Set max body size in bytes. + * + * @param maxBodySize the max body size + */ + public void setMaxBodySize(final Integer maxBodySize) { + this.maxBodySize = maxBodySize; + } + + /** + * Get batch size for uploading records. + * + * @return the batch size + */ + public int getBatchSize() { + return batchSize; + } + + /** + * Set batch size for uploading records. + * + * @param batchSize the batch size + */ + public void setBatchSize(final int batchSize) { + this.batchSize = batchSize; + } + + /** + * Get batch interval in milliseconds. + * + * @return the batch interval in milliseconds + */ + public long getBatchIntervalMs() { + return batchIntervalMs; + } + + /** + * Set batch interval in milliseconds. + * + * @param batchIntervalMs the batch interval in milliseconds + */ + public void setBatchIntervalMs(final long batchIntervalMs) { + this.batchIntervalMs = batchIntervalMs; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (Objects.isNull(o) || getClass() != o.getClass()) { + return false; + } + RecordConfig that = (RecordConfig) o; + return Objects.equals(maxBodySize, that.maxBodySize) + && Objects.equals(batchSize, that.batchSize) + && Objects.equals(batchIntervalMs, that.batchIntervalMs); + } + + @Override + public int hashCode() { + return Objects.hash(maxBodySize, batchSize, batchIntervalMs); + } + } +} diff --git a/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/config/ShenyuHttpRecordCenterConfig.java b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/config/ShenyuHttpRecordCenterConfig.java new file mode 100644 index 000000000000..e0b6a83e908f --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/config/ShenyuHttpRecordCenterConfig.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.config; + +import java.util.Properties; + +/** + * ShenyuHttpRecordCenterConfig. + * + *

Configuration for connecting to the Shenyu admin server + * for uploading recorded HTTP request/response data.

+ */ +public class ShenyuHttpRecordCenterConfig { + + private String serverLists; + + private Properties props = new Properties(); + + /** + * Get server lists. + * + * @return comma-separated server URLs + */ + public String getServerLists() { + return serverLists; + } + + /** + * Set server lists. + * + * @param serverLists comma-separated server URLs + */ + public void setServerLists(final String serverLists) { + this.serverLists = serverLists; + } + + /** + * Get props. + * + * @return additional properties (username, password, etc.) + */ + public Properties getProps() { + return props; + } + + /** + * Set props. + * + * @param props additional properties + */ + public void setProps(final Properties props) { + this.props = props; + } + +} diff --git a/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/entity/ShenyuHttpRequestRecord.java b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/entity/ShenyuHttpRequestRecord.java new file mode 100644 index 000000000000..54ae000fec41 --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/entity/ShenyuHttpRequestRecord.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.entity; + +import java.util.Map; + +/** + * ShenyuHttpRequestRecord. + * + *

Entity representing a recorded HTTP request and its response, + * used for capture and later replay.

+ */ +public class ShenyuHttpRequestRecord { + + private String taskId; + + private String traceId; + + private String method; + + private String requestUri; + + private String queryParams; + + private String requestBody; + + private Map requestHeaders; + + private Integer status; + + private Map responseHeaders; + + private String responseBody; + + /** + * Get task id. + * + * @return the task id + */ + public String getTaskId() { + return taskId; + } + + /** + * Set task id. + * + * @param taskId the task id + */ + public void setTaskId(final String taskId) { + this.taskId = taskId; + } + + /** + * Set request body. + * + * @param requestBody the request body + */ + public void setRequestBody(final String requestBody) { + this.requestBody = requestBody; + } + + /** + * Set method. + * + * @param method the HTTP method + */ + public void setMethod(final String method) { + this.method = method; + } + + /** + * Set request uri. + * + * @param requestUri the request URI + */ + public void setRequestUri(final String requestUri) { + this.requestUri = requestUri; + } + + /** + * Set query params. + * + * @param queryParams the query parameters + */ + public void setQueryParams(final String queryParams) { + this.queryParams = queryParams; + } + + /** + * Set request headers. + * + * @param requestHeaders the request headers + */ + public void setRequestHeaders(final Map requestHeaders) { + this.requestHeaders = requestHeaders; + } + + /** + * Set status. + * + * @param status the HTTP status code + */ + public void setStatus(final Integer status) { + this.status = status; + } + + /** + * Set response headers. + * + * @param responseHeaders the response headers + */ + public void setResponseHeaders(final Map responseHeaders) { + this.responseHeaders = responseHeaders; + } + + /** + * Set response body. + * + * @param responseBody the response body + */ + public void setResponseBody(final String responseBody) { + this.responseBody = responseBody; + } + + /** + * Set trace id. + * + * @param traceId the trace id + */ + public void setTraceId(final String traceId) { + this.traceId = traceId; + } + + /** + * Get trace id. + * + * @return the trace id + */ + public String getTraceId() { + return traceId; + } + + /** + * Get method. + * + * @return the HTTP method + */ + public String getMethod() { + return method; + } + + /** + * Get request uri. + * + * @return the request URI + */ + public String getRequestUri() { + return requestUri; + } + + /** + * Get query params. + * + * @return the query parameters + */ + public String getQueryParams() { + return queryParams; + } + + /** + * Get request headers. + * + * @return the request headers + */ + public Map getRequestHeaders() { + return requestHeaders; + } + + /** + * Get status. + * + * @return the HTTP status code + */ + public Integer getStatus() { + return status; + } + + /** + * Get response body. + * + * @return the response body + */ + public String getResponseBody() { + return responseBody; + } + + /** + * Get response headers. + * + * @return the response headers + */ + public Map getResponseHeaders() { + return responseHeaders; + } + + /** + * Get request body. + * + * @return the request body + */ + public String getRequestBody() { + return requestBody; + } + +} diff --git a/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/handler/HttpRecordPluginDataHandler.java b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/handler/HttpRecordPluginDataHandler.java new file mode 100644 index 000000000000..7e08b7629a6c --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/handler/HttpRecordPluginDataHandler.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.handler; + +import org.apache.shenyu.common.constant.Constants; +import org.apache.shenyu.common.dto.PluginData; +import org.apache.shenyu.common.dto.RuleData; +import org.apache.shenyu.common.dto.SelectorData; +import org.apache.shenyu.common.dto.convert.rule.HttpRecordHandle; +import org.apache.shenyu.common.enums.PluginEnum; +import org.apache.shenyu.common.utils.GsonUtils; +import org.apache.shenyu.plugin.base.cache.CommonHandleCache; +import org.apache.shenyu.plugin.base.handler.PluginDataHandler; +import org.apache.shenyu.plugin.base.utils.BeanHolder; +import org.apache.shenyu.plugin.base.utils.CacheKeyUtils; +import org.apache.shenyu.plugin.record.collector.HttpRecordCollector; +import org.apache.shenyu.plugin.record.config.HttpRecordCollectConfig; + +import java.util.Optional; +import java.util.function.Supplier; + +/** + * HttpRecordPluginDataHandler. + * + *

Handles plugin/selector/rule data updates from the admin server, + * caching the record handle config and starting/stopping the collector.

+ */ +public class HttpRecordPluginDataHandler implements PluginDataHandler { + + public static final Supplier> CACHED_HANDLE = + new BeanHolder<>(CommonHandleCache::new); + + @Override + public String pluginNamed() { + return PluginEnum.HTTP_RECORD.getName(); + } + + @Override + public void handlerPlugin(final PluginData pluginData) { + HttpRecordCollectConfig.RecordConfig recordConfig = + GsonUtils.getGson().fromJson(pluginData.getConfig(), HttpRecordCollectConfig.RecordConfig.class); + HttpRecordCollectConfig.INSTANCE.setRecordConfig(recordConfig); + HttpRecordCollector.INSTANCE.start(); + } + + @Override + public void removePlugin(final PluginData pluginData) { + HttpRecordCollector.INSTANCE.stop(); + PluginDataHandler.super.removePlugin(pluginData); + } + + @Override + public void handlerSelector(final SelectorData selectorData) { + CACHED_HANDLE.get().cachedHandle(CacheKeyUtils.INST.getKey(selectorData.getId(), Constants.DEFAULT_RULE), + HttpRecordHandle.newDefaultInstance()); + } + + @Override + public void removeSelector(final SelectorData selectorData) { + CACHED_HANDLE.get().removeHandle(CacheKeyUtils.INST.getKey(selectorData.getId(), Constants.DEFAULT_RULE)); + } + + @Override + public void handlerRule(final RuleData ruleData) { + Optional.ofNullable(ruleData.getHandle()).ifPresent(s -> { + final HttpRecordHandle httpRecordHandle = GsonUtils.getInstance().fromJson(s, HttpRecordHandle.class); + httpRecordHandle.setTaskId(String.format("%s_%s_%s", + httpRecordHandle.getStartTime(), httpRecordHandle.getEndTime(), ruleData.getId())); + CACHED_HANDLE.get().cachedHandle(CacheKeyUtils.INST.getKey(ruleData), httpRecordHandle); + }); + } + + @Override + public void removeRule(final RuleData ruleData) { + Optional.ofNullable(ruleData.getHandle()) + .ifPresent(s -> CACHED_HANDLE.get().removeHandle(CacheKeyUtils.INST.getKey(ruleData))); + } +} diff --git a/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/utils/OkHttpTools.java b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/utils/OkHttpTools.java new file mode 100644 index 000000000000..0a082a83f758 --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/utils/OkHttpTools.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.utils; + +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Headers; +import okhttp3.HttpUrl; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.apache.shenyu.common.constant.Constants; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +/** + * OkHttpTools. + */ +public final class OkHttpTools { + + /** + * The constant JSON. + */ + private static final MediaType JSON = MediaType.parse("application/json; charset=utf-8"); + + private static final OkHttpTools OK_HTTP_TOOLS = new OkHttpTools(); + + private final OkHttpClient client; + + private OkHttpTools() { + OkHttpClient.Builder builder = new OkHttpClient.Builder(); + builder.connectTimeout(10, TimeUnit.SECONDS); + builder.readTimeout(10, TimeUnit.SECONDS); + builder.writeTimeout(10, TimeUnit.SECONDS); + client = builder.build(); + } + + /** + * Gets instance. + * + * @return the instance + */ + public static OkHttpTools getInstance() { + return OK_HTTP_TOOLS; + } + + /** + * Post string. + * + * @param url the url + * @param json the json + * @return the string + * @throws IOException the io exception + */ + public String post(final String url, final String json) throws IOException { + RequestBody body = RequestBody.create(JSON, json); + Request request = new Request.Builder() + .url(url) + .post(body) + .build(); + try (Response response = client.newCall(request).execute()) { + ResponseBody responseBody = response.body(); + return Objects.nonNull(responseBody) ? responseBody.string() : ""; + } + } + + /** + * Post string. + * + * @param url the url + * @param json the json + * @param headers the headers + * @return the string + * @throws IOException the io exception + */ + public String post(final String url, final String json, final Headers headers) throws IOException { + RequestBody body = RequestBody.create(JSON, json); + Request request = new Request.Builder() + .headers(headers) + .url(url) + .post(body) + .build(); + try (Response response = client.newCall(request).execute()) { + ResponseBody responseBody = response.body(); + return Objects.nonNull(responseBody) ? responseBody.string() : ""; + } + } + + /** + * Get string. + * + * @param url the url + * @param query the query + * @return the http result + * @throws IOException the io exception + */ + public String get(final String url, final Map query) throws IOException { + Request.Builder reqBuild = new Request.Builder(); + HttpUrl.Builder urlBuilder = HttpUrl.parse(url).newBuilder(); + query.forEach((k, v) -> urlBuilder.addQueryParameter(k, String.valueOf(v))); + reqBuild.url(urlBuilder.build()); + Request request = reqBuild.build(); + try (Response response = client.newCall(request).execute()) { + ResponseBody responseBody = response.body(); + return Objects.nonNull(responseBody) ? responseBody.string() : ""; + } + } + + /** + * Get string by username and password. + * + * @param url the url + * @param userName the userName + * @param passWord the passWord + * @return the http result + * @throws IOException the io exception + */ + public String get(final String url, final String userName, final String passWord) throws IOException { + Request.Builder reqBuild = new Request.Builder(); + HttpUrl.Builder urlBuilder = HttpUrl.parse(url).newBuilder(); + urlBuilder.addQueryParameter(Constants.USER_NAME, userName); + urlBuilder.addQueryParameter(Constants.PASS_WORD, passWord); + reqBuild.url(urlBuilder.build()); + Request request = reqBuild.build(); + try (Response response = client.newCall(request).execute()) { + ResponseBody responseBody = response.body(); + return Objects.nonNull(responseBody) ? responseBody.string() : ""; + } + } + +} diff --git a/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/utils/RecordUtils.java b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/utils/RecordUtils.java new file mode 100644 index 000000000000..0a777ea766e3 --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/main/java/org/apache/shenyu/plugin/record/utils/RecordUtils.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.utils; + +import com.google.common.collect.Sets; +import org.springframework.http.HttpHeaders; + +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * RecordUtils. + * + *

Utility methods for the http record plugin, including + * content-type classification and header conversion.

+ */ +public class RecordUtils { + + private static final Set BINARY_TYPE_LIST = Sets.newHashSet("image", "multipart", "cbor", + "octet-stream", "pdf", "javascript", "css"); + + /** + * Judge whether is not binary type. + * + * @param headers request or response header + * @return whether not binary type + */ + public static boolean isNotBinaryType(final HttpHeaders headers) { + return Optional.ofNullable(headers).map(HttpHeaders::getContentType) + .map(contentType -> !BINARY_TYPE_LIST.contains(contentType.getType()) + && !BINARY_TYPE_LIST.contains(contentType.getSubtype())) + .orElse(true); + } + + /** + * Get request header map. Returns empty map if headers is null. + * + * @param headers request headers + * @return header map with joined multi-values + */ + public static Map getHeaders(final HttpHeaders headers) { + if (Objects.isNull(headers)) { + return Map.of(); + } + return headers.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> String.join(",", entry.getValue()))); + } +} diff --git a/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/HttpRecordPluginTest.java b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/HttpRecordPluginTest.java new file mode 100644 index 000000000000..90b6742b6c13 --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/HttpRecordPluginTest.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record; + +import org.apache.shenyu.common.constant.Constants; +import org.apache.shenyu.common.dto.RuleData; +import org.apache.shenyu.common.dto.SelectorData; +import org.apache.shenyu.common.dto.convert.rule.HttpRecordHandle; +import org.apache.shenyu.common.enums.PluginEnum; +import org.apache.shenyu.plugin.api.ShenyuPluginChain; +import org.apache.shenyu.plugin.base.utils.CacheKeyUtils; +import org.apache.shenyu.plugin.record.body.RecordServerHttpRequest; +import org.apache.shenyu.plugin.record.body.RecordServerHttpResponse; +import org.apache.shenyu.plugin.record.config.HttpRecordCollectConfig; +import org.apache.shenyu.plugin.record.entity.ShenyuHttpRequestRecord; +import org.apache.shenyu.plugin.record.handler.HttpRecordPluginDataHandler; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.mock.http.server.reactive.MockServerHttpRequest; +import org.springframework.mock.web.server.MockServerWebExchange; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import java.lang.reflect.Field; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public final class HttpRecordPluginTest { + + private HttpRecordPlugin httpRecordPlugin; + + private ServerWebExchange exchange; + + private ShenyuPluginChain chain; + + private SelectorData selectorData; + + private RuleData ruleData; + + private ArgumentCaptor exchangeCaptor; + + @BeforeEach + public void setUp() { + HttpRecordCollectConfig.INSTANCE.setRecordConfig(new HttpRecordCollectConfig.RecordConfig()); + httpRecordPlugin = new HttpRecordPlugin(); + exchange = MockServerWebExchange.from(MockServerHttpRequest.get("http://localhost/test").build()); + chain = mock(ShenyuPluginChain.class); + when(chain.execute(any())).thenReturn(Mono.empty()); + selectorData = SelectorData.builder().id("selectorId").build(); + ruleData = new RuleData(); + ruleData.setId("ruleId"); + ruleData.setSelectorId("selectorId"); + exchangeCaptor = ArgumentCaptor.forClass(ServerWebExchange.class); + } + + @Test + public void testNamed() { + assertEquals(PluginEnum.HTTP_RECORD.getName(), httpRecordPlugin.named()); + } + + @Test + public void testGetOrder() { + assertEquals(PluginEnum.HTTP_RECORD.getCode(), httpRecordPlugin.getOrder()); + } + + @Test + public void testDoExecuteWithNullHandle() { + exchange = MockServerWebExchange.from(MockServerHttpRequest.get("http://localhost/test?test=test") + .header("X-test", "test") + .build()); + HttpRecordPluginDataHandler.CACHED_HANDLE.get().removeHandle(CacheKeyUtils.INST.getKey(ruleData)); + Mono result = httpRecordPlugin.doExecute(exchange, chain, selectorData, ruleData); + StepVerifier.create(result).expectSubscription().verifyComplete(); + verify(chain).execute(exchangeCaptor.capture()); + assertFalse(exchangeCaptor.getValue().getRequest() instanceof RecordServerHttpRequest); + } + + @Test + public void testDoExecuteWithReplayHeader() { + HttpRecordHandle handle = new HttpRecordHandle(); + long now = System.currentTimeMillis(); + handle.setStartTime(now - 60000); + handle.setEndTime(now + 60000); + HttpRecordPluginDataHandler.CACHED_HANDLE.get().cachedHandle(CacheKeyUtils.INST.getKey(ruleData), handle); + + ServerWebExchange replayExchange = MockServerWebExchange.from( + MockServerHttpRequest.get("http://localhost/test") + .header(Constants.X_SHENYU_REPLAY, "true") + .build()); + Mono result = httpRecordPlugin.doExecute(replayExchange, chain, selectorData, ruleData); + StepVerifier.create(result).expectSubscription().verifyComplete(); + verify(chain).execute(exchangeCaptor.capture()); + assertFalse(exchangeCaptor.getValue().getRequest() instanceof RecordServerHttpRequest); + } + + @Test + public void testDoExecuteWithTimeWindowExpired() { + HttpRecordHandle handle = new HttpRecordHandle(); + handle.setStartTime(System.currentTimeMillis() - 120000); + handle.setEndTime(System.currentTimeMillis() - 60000); + HttpRecordPluginDataHandler.CACHED_HANDLE.get().cachedHandle(CacheKeyUtils.INST.getKey(ruleData), handle); + exchange = MockServerWebExchange.from(MockServerHttpRequest.get("http://localhost/test") + .build()); + Mono result = httpRecordPlugin.doExecute(exchange, chain, selectorData, ruleData); + StepVerifier.create(result).expectSubscription().verifyComplete(); + verify(chain).execute(exchangeCaptor.capture()); + assertFalse(exchangeCaptor.getValue().getRequest() instanceof RecordServerHttpRequest); + } + + @Test + public void testDoExecuteWithinTimeWindow() { + HttpRecordHandle handle = new HttpRecordHandle(); + long now = System.currentTimeMillis(); + handle.setStartTime(now - 60000); + handle.setEndTime(now + 60000); + HttpRecordPluginDataHandler.CACHED_HANDLE.get().cachedHandle(CacheKeyUtils.INST.getKey(ruleData), handle); + exchange = MockServerWebExchange.from(MockServerHttpRequest.get("http://localhost/test") + .header("test", "test") + .build()); + + Mono result = httpRecordPlugin.doExecute(exchange, chain, selectorData, ruleData); + StepVerifier.create(result).expectSubscription().verifyComplete(); + verify(chain).execute(exchangeCaptor.capture()); + ServerWebExchange captured = exchangeCaptor.getValue(); + + assertTrue(captured.getRequest() instanceof RecordServerHttpRequest, + "Request should be wrapped with RecordServerHttpRequest within time window"); + assertTrue(captured.getResponse() instanceof RecordServerHttpResponse, + "Response should be wrapped with RecordServerHttpResponse within time window"); + + ShenyuHttpRequestRecord record = extractRecord((RecordServerHttpRequest) captured.getRequest()); + assertNotNull(record.getTraceId(), "traceId should be generated"); + assertFalse(record.getTraceId().isEmpty(), "traceId should not be empty"); + assertEquals("GET", record.getMethod(), "method should be GET"); + assertEquals("/test", record.getRequestUri(), "requestUri should be /test"); + assertNull(record.getQueryParams(), "queryParams should be null for request without query string"); + assertNotNull(record.getRequestHeaders(), "requestHeaders should not be null"); + assertFalse(record.getRequestHeaders().isEmpty(), "requestHeaders should not be empty"); + } + + @Test + public void testDoExecuteWithinTimeWindowWithQueryParams() { + HttpRecordHandle handle = new HttpRecordHandle(); + long now = System.currentTimeMillis(); + handle.setStartTime(now - 60000); + handle.setEndTime(now + 60000); + HttpRecordPluginDataHandler.CACHED_HANDLE.get().cachedHandle(CacheKeyUtils.INST.getKey(ruleData), handle); + + ServerWebExchange queryExchange = MockServerWebExchange.from( + MockServerHttpRequest.get("http://localhost/test") + .queryParam("foo", "bar") + .queryParam("baz", "qux") + .build()); + + Mono result = httpRecordPlugin.doExecute(queryExchange, chain, selectorData, ruleData); + StepVerifier.create(result).expectSubscription().verifyComplete(); + verify(chain).execute(exchangeCaptor.capture()); + ServerWebExchange captured = exchangeCaptor.getValue(); + ShenyuHttpRequestRecord record = extractRecord((RecordServerHttpRequest) captured.getRequest()); + + assertEquals("foo=bar&baz=qux", record.getQueryParams(), "queryParams should capture the full query string"); + assertEquals("/test", record.getRequestUri(), "requestUri should be /test"); + } + + @Test + public void testDoExecuteWithinTimeWindowWithTaskId() { + HttpRecordHandle handle = new HttpRecordHandle(); + long now = System.currentTimeMillis(); + handle.setStartTime(now - 60000); + handle.setEndTime(now + 60000); + handle.setTaskId("myTaskId123"); + HttpRecordPluginDataHandler.CACHED_HANDLE.get().cachedHandle(CacheKeyUtils.INST.getKey(ruleData), handle); + + exchange = MockServerWebExchange.from(MockServerHttpRequest.get("http://localhost/test") + .build()); + + Mono result = httpRecordPlugin.doExecute(exchange, chain, selectorData, ruleData); + StepVerifier.create(result).expectSubscription().verifyComplete(); + verify(chain).execute(exchangeCaptor.capture()); + ServerWebExchange captured = exchangeCaptor.getValue(); + ShenyuHttpRequestRecord record = extractRecord((RecordServerHttpRequest) captured.getRequest()); + + assertEquals("myTaskId123", record.getTaskId(), "taskId should match the handle's taskId"); + } + + @Test + public void testDoExecuteWithinTimeWindowWithCustomHeaders() { + HttpRecordHandle handle = new HttpRecordHandle(); + long now = System.currentTimeMillis(); + handle.setStartTime(now - 60000); + handle.setEndTime(now + 60000); + HttpRecordPluginDataHandler.CACHED_HANDLE.get().cachedHandle(CacheKeyUtils.INST.getKey(ruleData), handle); + + ServerWebExchange headerExchange = MockServerWebExchange.from( + MockServerHttpRequest.get("http://localhost/test") + .header("X-Custom-Header", "customValue") + .header("Accept", "application/json") + .build()); + + Mono result = httpRecordPlugin.doExecute(headerExchange, chain, selectorData, ruleData); + StepVerifier.create(result).expectSubscription().verifyComplete(); + verify(chain).execute(exchangeCaptor.capture()); + ServerWebExchange captured = exchangeCaptor.getValue(); + ShenyuHttpRequestRecord record = extractRecord((RecordServerHttpRequest) captured.getRequest()); + + assertNotNull(record.getRequestHeaders(), "requestHeaders should not be null"); + assertTrue(record.getRequestHeaders().containsKey("X-Custom-Header"), + "requestHeaders should contain X-Custom-Header"); + assertEquals("customValue", record.getRequestHeaders().get("X-Custom-Header"), + "X-Custom-Header value should match"); + assertTrue(record.getRequestHeaders().containsKey("Accept"), + "requestHeaders should contain Accept header"); + } + + @Test + public void testDoExecuteWithStartTimeNotYetReached() { + HttpRecordHandle handle = new HttpRecordHandle(); + long now = System.currentTimeMillis(); + handle.setStartTime(now + 60000); + handle.setEndTime(now + 120000); + HttpRecordPluginDataHandler.CACHED_HANDLE.get().cachedHandle(CacheKeyUtils.INST.getKey(ruleData), handle); + exchange = MockServerWebExchange.from(MockServerHttpRequest.get("http://localhost/test") + .build()); + Mono result = httpRecordPlugin.doExecute(exchange, chain, selectorData, ruleData); + StepVerifier.create(result).expectSubscription().verifyComplete(); + verify(chain).execute(exchangeCaptor.capture()); + assertFalse(exchangeCaptor.getValue().getRequest() instanceof RecordServerHttpRequest); + } + + @Test + public void testDoExecuteWithinTimeWindowEachCallGeneratesUniqueTraceId() { + HttpRecordHandle handle = new HttpRecordHandle(); + long now = System.currentTimeMillis(); + handle.setStartTime(now - 60000); + handle.setEndTime(now + 60000); + HttpRecordPluginDataHandler.CACHED_HANDLE.get().cachedHandle(CacheKeyUtils.INST.getKey(ruleData), handle); + + exchange = MockServerWebExchange.from(MockServerHttpRequest.get("http://localhost/test?test=test") + .build()); + + Mono result1 = httpRecordPlugin.doExecute(exchange, chain, selectorData, ruleData); + StepVerifier.create(result1).expectSubscription().verifyComplete(); + + Mono result2 = httpRecordPlugin.doExecute(exchange, chain, selectorData, ruleData); + StepVerifier.create(result2).expectSubscription().verifyComplete(); + + verify(chain, times(2)).execute(exchangeCaptor.capture()); + + ShenyuHttpRequestRecord record1 = extractRecord((RecordServerHttpRequest) exchangeCaptor.getAllValues().get(0).getRequest()); + ShenyuHttpRequestRecord record2 = extractRecord((RecordServerHttpRequest) exchangeCaptor.getAllValues().get(1).getRequest()); + + assertNotNull(record1.getTraceId()); + assertNotNull(record2.getTraceId()); + assertNotEquals(record1.getTraceId(), record2.getTraceId(), + "Each request should get a unique traceId"); + } + + private ShenyuHttpRequestRecord extractRecord(final RecordServerHttpRequest recordRequest) { + try { + Field recordField = RecordServerHttpRequest.class.getDeclaredField("record"); + recordField.setAccessible(true); + return (ShenyuHttpRequestRecord) recordField.get(recordRequest); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException("Failed to extract record from RecordServerHttpRequest", e); + } + } +} diff --git a/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/body/BodyWriterTest.java b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/body/BodyWriterTest.java new file mode 100644 index 000000000000..c334dfa3908f --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/body/BodyWriterTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.body; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public final class BodyWriterTest { + + private static final String TEST_DATA = "hello shenyu"; + + private BodyWriter bodyWriter; + + @BeforeEach + public void setUp() { + bodyWriter = new BodyWriter(); + } + + @Test + public void testWriteAndOutput() { + bodyWriter.write(ByteBuffer.wrap(TEST_DATA.getBytes(StandardCharsets.UTF_8))); + assertFalse(bodyWriter.isEmpty()); + assertEquals(TEST_DATA.length(), bodyWriter.size()); + assertEquals(TEST_DATA, bodyWriter.output()); + } + + @Test + public void testEmptyWriterProperties() { + assertTrue(bodyWriter.isEmpty()); + assertEquals(0, bodyWriter.size()); + assertFalse(bodyWriter.isSizeExceeded()); + } + + @Test + public void testMaxSizeExceeded() { + int maxSize = 5; + BodyWriter limitedWriter = new BodyWriter(maxSize); + limitedWriter.write(ByteBuffer.wrap("hello world".getBytes(StandardCharsets.UTF_8))); + assertTrue(limitedWriter.isSizeExceeded()); + } + + @Test + public void testMaxSizeExactWriteDoesNotExceed() { + int maxSize = 5; + BodyWriter limitedWriter = new BodyWriter(maxSize); + limitedWriter.write(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8))); + assertFalse(limitedWriter.isSizeExceeded()); + assertEquals(5, limitedWriter.size()); + } + + @Test + public void testWriteAfterSizeExceededIsDropped() { + int maxSize = 5; + BodyWriter limitedWriter = new BodyWriter(maxSize); + limitedWriter.write(ByteBuffer.wrap("hello world".getBytes(StandardCharsets.UTF_8))); + assertTrue(limitedWriter.isSizeExceeded()); + int sizeBefore = limitedWriter.size(); + limitedWriter.write(ByteBuffer.wrap("extra".getBytes(StandardCharsets.UTF_8))); + assertEquals(sizeBefore, limitedWriter.size()); + } + + @Test + public void testMultipleWritesConcatenates() { + bodyWriter.write(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8))); + bodyWriter.write(ByteBuffer.wrap(" shenyu".getBytes(StandardCharsets.UTF_8))); + assertEquals(TEST_DATA, bodyWriter.output()); + } + + @Test + public void testBelowLimitNotExceeded() { + BodyWriter limitedWriter = new BodyWriter(1024); + limitedWriter.write(ByteBuffer.wrap(TEST_DATA.getBytes(StandardCharsets.UTF_8))); + assertFalse(limitedWriter.isSizeExceeded()); + } + + @Test + public void testDefaultConstructorNoLimit() { + BodyWriter unlimitedWriter = new BodyWriter(); + byte[] largeData = new byte[1024]; + unlimitedWriter.write(ByteBuffer.wrap(largeData)); + assertFalse(unlimitedWriter.isSizeExceeded()); + assertEquals(1024, unlimitedWriter.size()); + } + + @Test + public void testOutputPreventsFurtherWrites() { + bodyWriter.write(ByteBuffer.wrap(TEST_DATA.getBytes(StandardCharsets.UTF_8))); + String result = bodyWriter.output(); + assertEquals(TEST_DATA, result); + int sizeBefore = bodyWriter.size(); + bodyWriter.write(ByteBuffer.wrap("extra".getBytes(StandardCharsets.UTF_8))); + assertEquals(sizeBefore, bodyWriter.size()); + } + + @Test + public void testOutputReturnsEmptyStringWhenEmpty() { + assertEquals("", bodyWriter.output()); + } +} \ No newline at end of file diff --git a/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/body/RecordServerHttpRequestTest.java b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/body/RecordServerHttpRequestTest.java new file mode 100644 index 000000000000..caf4cadff01e --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/body/RecordServerHttpRequestTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.body; + +import org.apache.shenyu.plugin.record.config.HttpRecordCollectConfig; +import org.apache.shenyu.plugin.record.entity.ShenyuHttpRequestRecord; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.core.io.buffer.DefaultDataBufferFactory; +import org.springframework.http.server.reactive.ServerHttpRequest; +import org.springframework.mock.http.server.reactive.MockServerHttpRequest; +import reactor.core.publisher.Flux; + +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public final class RecordServerHttpRequestTest { + + private DefaultDataBufferFactory bufferFactory; + + @BeforeEach + public void setUp() { + bufferFactory = new DefaultDataBufferFactory(); + HttpRecordCollectConfig.INSTANCE.setRecordConfig(new HttpRecordCollectConfig.RecordConfig()); + } + + @Test + public void testGetBodyWithPostRequest() { + String jsonBody = "{\"key\":\"value\"}"; + MockServerHttpRequest mockRequest = MockServerHttpRequest.post("http://localhost/test") + .contentType(org.springframework.http.MediaType.APPLICATION_JSON) + .body(jsonBody); + ShenyuHttpRequestRecord record = new ShenyuHttpRequestRecord(); + RecordServerHttpRequest recordRequest = + new RecordServerHttpRequest(mockRequest, record); + + Flux bodyFlux = recordRequest.getBody(); + StringBuilder sb = new StringBuilder(); + bodyFlux.doOnNext(dataBuffer -> { + byte[] bytes = new byte[dataBuffer.readableByteCount()]; + dataBuffer.read(bytes); + sb.append(new String(bytes, StandardCharsets.UTF_8)); + }).blockLast(); + + assertEquals(jsonBody, sb.toString()); + } + + @Test + public void testGetBodyTruncatesOversizedBody() { + HttpRecordCollectConfig.RecordConfig config = new HttpRecordCollectConfig.RecordConfig(); + config.setMaxBodySize(10); + HttpRecordCollectConfig.INSTANCE.setRecordConfig(config); + + String bodyContent = "this is a very long body that exceeds the max size limit"; + MockServerHttpRequest mockRequest = MockServerHttpRequest.post("http://localhost/test") + .contentType(org.springframework.http.MediaType.APPLICATION_JSON) + .body(bodyContent); + ShenyuHttpRequestRecord record = new ShenyuHttpRequestRecord(); + RecordServerHttpRequest recordRequest = + new RecordServerHttpRequest(mockRequest, record); + + Flux bodyFlux = recordRequest.getBody(); + bodyFlux.doOnNext(DataBuffer::readableByteCount).blockLast(); + + assertTrue(record.getRequestBody().startsWith("[TRUNCATED]")); + + HttpRecordCollectConfig.INSTANCE.setRecordConfig(new HttpRecordCollectConfig.RecordConfig()); + } + + @Test + public void testDelegatePreservesOriginalRequestProperties() { + MockServerHttpRequest mockRequest = MockServerHttpRequest.get("http://localhost/test?param=value") + .header("X-Custom", "test-value") + .build(); + ShenyuHttpRequestRecord record = new ShenyuHttpRequestRecord(); + RecordServerHttpRequest recordRequest = + new RecordServerHttpRequest(mockRequest, record); + + assertEquals("GET", recordRequest.getMethod().name()); + assertEquals("/test", recordRequest.getURI().getPath()); + assertEquals("param=value", recordRequest.getURI().getQuery()); + assertEquals("test-value", recordRequest.getHeaders().get("X-Custom").get(0)); + } + + @Test + public void testGetBodyWithEmptyBody() { + MockServerHttpRequest mockRequest = MockServerHttpRequest.get("http://localhost/test").build(); + ShenyuHttpRequestRecord record = new ShenyuHttpRequestRecord(); + RecordServerHttpRequest recordRequest = + new RecordServerHttpRequest(mockRequest, record); + + ServerHttpRequest delegate = recordRequest.getDelegate(); + assertEquals(mockRequest, delegate); + } + + @Test + public void testGetBodyCapturesRequestBodyViaDoFinally() { + String bodyContent = "hello"; + MockServerHttpRequest mockRequest = MockServerHttpRequest.post("http://localhost/test") + .contentType(org.springframework.http.MediaType.APPLICATION_JSON) + .body(bodyContent); + ShenyuHttpRequestRecord record = new ShenyuHttpRequestRecord(); + RecordServerHttpRequest recordRequest = + new RecordServerHttpRequest(mockRequest, record); + + recordRequest.getBody().blockLast(); + + String capturedBody = record.getRequestBody(); + assertEquals(bodyContent, capturedBody); + } +} \ No newline at end of file diff --git a/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/body/RecordServerHttpResponseTest.java b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/body/RecordServerHttpResponseTest.java new file mode 100644 index 000000000000..1c9f497f59be --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/body/RecordServerHttpResponseTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.body; + +import org.apache.shenyu.plugin.record.collector.HttpRecordCollector; +import org.apache.shenyu.plugin.record.config.HttpRecordCollectConfig; +import org.apache.shenyu.plugin.record.entity.ShenyuHttpRequestRecord; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.core.io.buffer.DefaultDataBufferFactory; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatusCode; +import org.springframework.http.MediaType; +import org.springframework.http.server.reactive.ServerHttpResponse; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public final class RecordServerHttpResponseTest { + + private DefaultDataBufferFactory bufferFactory; + + @BeforeEach + public void setUp() { + bufferFactory = new DefaultDataBufferFactory(); + HttpRecordCollectConfig.INSTANCE.setRecordConfig(new HttpRecordCollectConfig.RecordConfig()); + } + + @AfterEach + public void tearDown() { + HttpRecordCollectConfig.INSTANCE.setRecordConfig(new HttpRecordCollectConfig.RecordConfig()); + } + + @Test + public void testWriteWithJsonResponseCapturesStatusAndHeaders() { + ServerHttpResponse mockResponse = mock(ServerHttpResponse.class); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + when(mockResponse.getHeaders()).thenReturn(headers); + when(mockResponse.getStatusCode()).thenReturn(HttpStatusCode.valueOf(200)); + when(mockResponse.writeWith(any())).thenAnswer(invocation -> { + Flux body = invocation.getArgument(0); + return body.then(Mono.empty()); + }); + + ServerWebExchange exchange = mock(ServerWebExchange.class); + when(exchange.getResponse()).thenReturn(mockResponse); + + ShenyuHttpRequestRecord record = new ShenyuHttpRequestRecord(); + TestableRecordServerHttpResponse response = new TestableRecordServerHttpResponse( + mockResponse, record, exchange); + + String responseBody = "{\"result\":\"ok\"}"; + DataBuffer dataBuffer = bufferFactory.wrap(responseBody.getBytes()); + Mono result = response.writeWith(Flux.just(dataBuffer)); + StepVerifier.create(result).expectSubscription().verifyComplete(); + + assertEquals(200, record.getStatus()); + assertTrue(record.getResponseHeaders().containsKey("Content-Type")); + assertEquals(responseBody, record.getResponseBody()); + assertEquals("done collect", record.getTaskId()); + } + + @Test + public void testWriteWithBinaryResponseCollectsWithoutBody() { + ServerHttpResponse mockResponse = mock(ServerHttpResponse.class); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.IMAGE_PNG); + when(mockResponse.getHeaders()).thenReturn(headers); + when(mockResponse.getStatusCode()).thenReturn(HttpStatusCode.valueOf(200)); + when(mockResponse.writeWith(any())).thenAnswer(invocation -> { + Flux body = invocation.getArgument(0); + return body.then(Mono.empty()); + }); + + ServerWebExchange exchange = mock(ServerWebExchange.class); + when(exchange.getResponse()).thenReturn(mockResponse); + + ShenyuHttpRequestRecord record = new ShenyuHttpRequestRecord(); + TestableRecordServerHttpResponse response = new TestableRecordServerHttpResponse( + mockResponse, record, exchange); + + byte[] imageData = new byte[]{(byte) 0x89, (byte) 0x50, (byte) 0x4E, (byte) 0x47}; + DataBuffer dataBuffer = bufferFactory.wrap(imageData); + Mono result = response.writeWith(Flux.just(dataBuffer)); + StepVerifier.create(result).expectSubscription().verifyComplete(); + + assertEquals(200, record.getStatus()); + assertNull(record.getResponseBody()); + assertEquals("done collect", record.getTaskId()); + } + + @Test + public void testWriteWithTruncatedBody() { + HttpRecordCollectConfig.RecordConfig config = new HttpRecordCollectConfig.RecordConfig(); + config.setMaxBodySize(10); + HttpRecordCollectConfig.INSTANCE.setRecordConfig(config); + + ServerHttpResponse mockResponse = mock(ServerHttpResponse.class); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + when(mockResponse.getHeaders()).thenReturn(headers); + when(mockResponse.getStatusCode()).thenReturn(HttpStatusCode.valueOf(200)); + when(mockResponse.writeWith(any())).thenAnswer(invocation -> { + Flux body = invocation.getArgument(0); + return body.then(Mono.empty()); + }); + + ServerWebExchange exchange = mock(ServerWebExchange.class); + when(exchange.getResponse()).thenReturn(mockResponse); + + ShenyuHttpRequestRecord record = new ShenyuHttpRequestRecord(); + TestableRecordServerHttpResponse response = new TestableRecordServerHttpResponse( + mockResponse, record, exchange); + + String longBody = "this is a very long response body that exceeds the max size limit"; + DataBuffer dataBuffer = bufferFactory.wrap(longBody.getBytes()); + Mono result = response.writeWith(Flux.just(dataBuffer)); + StepVerifier.create(result).expectSubscription().verifyComplete(); + + assertTrue(record.getResponseBody().startsWith("[TRUNCATED]")); + } + + private static class TestableRecordServerHttpResponse extends RecordServerHttpResponse { + + TestableRecordServerHttpResponse(final ServerHttpResponse delegate, + final ShenyuHttpRequestRecord record, + final ServerWebExchange exchange) { + super(delegate, record, new TestHttpRecordCollector(), exchange); + } + } + + private static class TestHttpRecordCollector extends HttpRecordCollector { + + @Override + public void collect(final ShenyuHttpRequestRecord record) { + record.setTaskId("done collect"); + } + } +} \ No newline at end of file diff --git a/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/collector/HttpRecordCollectorTest.java b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/collector/HttpRecordCollectorTest.java new file mode 100644 index 000000000000..b0c1eed55b23 --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/collector/HttpRecordCollectorTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.collector; + +import org.apache.shenyu.plugin.record.config.HttpRecordCollectConfig; +import org.apache.shenyu.plugin.record.entity.ShenyuHttpRequestRecord; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +public final class HttpRecordCollectorTest { + + private HttpRecordCollector collector; + + @BeforeEach + public void setUp() throws Exception { + collector = HttpRecordCollector.INSTANCE; + HttpRecordCollectConfig.INSTANCE.setRecordConfig(new HttpRecordCollectConfig.RecordConfig()); + } + + @AfterEach + public void tearDown() throws Exception { + setField(collector, "bufferQueue", null); + AtomicInteger bufferSize = getField(collector, "bufferSize"); + bufferSize.set(0); + setField(collector, "maxBufferSize", 0); + } + + @Test + public void testGetInstance() { + HttpRecordCollector instance1 = HttpRecordCollector.getInstance(); + HttpRecordCollector instance2 = HttpRecordCollector.getInstance(); + assertNotNull(instance1); + assertSame(instance1, instance2); + } + + @Test + public void testCollectWithNullRecord() { + collector.collect(null); + } + + @Test + public void testCollectWithNullBufferQueue() { + ShenyuHttpRequestRecord record = new ShenyuHttpRequestRecord(); + record.setTraceId("test-trace-id"); + collector.collect(record); + } + + @Test + public void testCollectPutsRecordIntoQueue() throws Exception { + ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); + setField(collector, "bufferQueue", queue); + setField(collector, "maxBufferSize", 100); + + ShenyuHttpRequestRecord record = new ShenyuHttpRequestRecord(); + record.setTraceId("test-trace-id"); + record.setRequestUri("/test"); + record.setMethod("GET"); + collector.collect(record); + + assertEquals(1, queue.size()); + assertEquals("test-trace-id", queue.peek().getTraceId()); + } + + @Test + public void testCollectDropsRecordWhenBufferFull() throws Exception { + ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); + setField(collector, "bufferQueue", queue); + setField(collector, "maxBufferSize", 1); + + ShenyuHttpRequestRecord record1 = new ShenyuHttpRequestRecord(); + record1.setTraceId("first"); + collector.collect(record1); + + ShenyuHttpRequestRecord record2 = new ShenyuHttpRequestRecord(); + record2.setTraceId("dropped"); + collector.collect(record2); + + assertEquals(1, queue.size()); + assertEquals("first", queue.peek().getTraceId()); + } + + @Test + public void testStopWithoutStart() { + collector.stop(); + } + + + @Test + public void testBatchSizeDefault() { + HttpRecordCollectConfig.RecordConfig config = new HttpRecordCollectConfig.RecordConfig(); + assertEquals(100, config.getBatchSize()); + } + + @Test + public void testBatchIntervalDefault() { + HttpRecordCollectConfig.RecordConfig config = new HttpRecordCollectConfig.RecordConfig(); + assertEquals(5000L, config.getBatchIntervalMs()); + } + + @Test + public void testMaxBodySizeDefault() { + HttpRecordCollectConfig.RecordConfig config = new HttpRecordCollectConfig.RecordConfig(); + assertEquals(524288, config.getMaxBodySize()); + } + + @SuppressWarnings("unchecked") + private static T getField(final Object target, final String fieldName) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + return (T) field.get(target); + } + + private static void setField(final Object target, final String fieldName, final Object value) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } +} \ No newline at end of file diff --git a/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/handler/HttpRecordPluginDataHandlerTest.java b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/handler/HttpRecordPluginDataHandlerTest.java new file mode 100644 index 000000000000..cec07bcc08db --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/handler/HttpRecordPluginDataHandlerTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.handler; + +import org.apache.shenyu.common.constant.Constants; +import org.apache.shenyu.common.dto.RuleData; +import org.apache.shenyu.common.dto.SelectorData; +import org.apache.shenyu.common.dto.convert.rule.HttpRecordHandle; +import org.apache.shenyu.common.enums.PluginEnum; +import org.apache.shenyu.common.utils.GsonUtils; +import org.apache.shenyu.plugin.base.utils.CacheKeyUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +public final class HttpRecordPluginDataHandlerTest { + + private HttpRecordPluginDataHandler handler; + + @BeforeEach + public void setUp() { + handler = new HttpRecordPluginDataHandler(); + } + + @Test + public void testPluginNamed() { + assertEquals(PluginEnum.HTTP_RECORD.getName(), handler.pluginNamed()); + } + + @Test + public void testHandlerRule() { + RuleData ruleData = new RuleData(); + ruleData.setId("ruleId"); + ruleData.setSelectorId("selectorId"); + long startTime = System.currentTimeMillis(); + long endTime = startTime + 60000; + HttpRecordHandle httpRecordHandle = new HttpRecordHandle(); + httpRecordHandle.setStartTime(startTime); + httpRecordHandle.setEndTime(endTime); + ruleData.setHandle(GsonUtils.getInstance().toJson(httpRecordHandle)); + handler.handlerRule(ruleData); + HttpRecordHandle cachedHandle = HttpRecordPluginDataHandler.CACHED_HANDLE.get() + .obtainHandle(CacheKeyUtils.INST.getKey(ruleData)); + assertNotNull(cachedHandle); + String expectedTaskId = startTime + "_" + endTime + "_" + ruleData.getId(); + assertEquals(expectedTaskId, cachedHandle.getTaskId()); + assertEquals(startTime, cachedHandle.getStartTime()); + assertEquals(endTime, cachedHandle.getEndTime()); + + handler.removeRule(ruleData); + assertNull(HttpRecordPluginDataHandler.CACHED_HANDLE.get() + .obtainHandle(CacheKeyUtils.INST.getKey(ruleData))); + } + + @Test + public void testHandlerRuleWithNullHandle() { + RuleData ruleData = new RuleData(); + ruleData.setId("ruleId2"); + ruleData.setSelectorId("selectorId2"); + ruleData.setHandle(null); + handler.handlerRule(ruleData); + assertNull(HttpRecordPluginDataHandler.CACHED_HANDLE.get() + .obtainHandle(CacheKeyUtils.INST.getKey(ruleData))); + } + + @Test + public void testHandlerSelector() { + SelectorData selectorData = SelectorData.builder() + .id("selectorId") + .build(); + handler.handlerSelector(selectorData); + HttpRecordHandle handle = HttpRecordPluginDataHandler.CACHED_HANDLE.get() + .obtainHandle(CacheKeyUtils.INST.getKey(selectorData.getId(), Constants.DEFAULT_RULE)); + assertNotNull(handle); + assertNotNull(handle.getStartTime()); + assertNotNull(handle.getEndTime()); + + handler.removeSelector(selectorData); + assertNull(HttpRecordPluginDataHandler.CACHED_HANDLE.get() + .obtainHandle(CacheKeyUtils.INST.getKey(selectorData.getId(), Constants.DEFAULT_RULE))); + } +} \ No newline at end of file diff --git a/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/utils/RecordUtilsTest.java b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/utils/RecordUtilsTest.java new file mode 100644 index 000000000000..e8de15a32132 --- /dev/null +++ b/shenyu-plugin/shenyu-plugin-record/src/test/java/org/apache/shenyu/plugin/record/utils/RecordUtilsTest.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.plugin.record.utils; + +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpHeaders; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public final class RecordUtilsTest { + + @Test + public void testIsNotBinaryTypeWithJson() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(org.springframework.http.MediaType.APPLICATION_JSON); + assertTrue(RecordUtils.isNotBinaryType(headers)); + } + + @Test + public void testIsNotBinaryTypeWithPlainText() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(org.springframework.http.MediaType.TEXT_PLAIN); + assertTrue(RecordUtils.isNotBinaryType(headers)); + } + + @Test + public void testIsNotBinaryTypeWithHtml() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(org.springframework.http.MediaType.TEXT_HTML); + assertTrue(RecordUtils.isNotBinaryType(headers)); + } + + @Test + public void testIsNotBinaryTypeWithImage() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(org.springframework.http.MediaType.IMAGE_PNG); + assertFalse(RecordUtils.isNotBinaryType(headers)); + } + + @Test + public void testIsNotBinaryTypeWithOctetStream() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(org.springframework.http.MediaType.APPLICATION_OCTET_STREAM); + assertFalse(RecordUtils.isNotBinaryType(headers)); + } + + @Test + public void testIsNotBinaryTypeWithPdf() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(org.springframework.http.MediaType.APPLICATION_PDF); + assertFalse(RecordUtils.isNotBinaryType(headers)); + } + + @Test + public void testIsNotBinaryTypeWithMultipart() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(org.springframework.http.MediaType.MULTIPART_FORM_DATA); + assertFalse(RecordUtils.isNotBinaryType(headers)); + } + + @Test + public void testIsNotBinaryTypeWithJavascript() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(org.springframework.http.MediaType.valueOf("application/javascript")); + assertFalse(RecordUtils.isNotBinaryType(headers)); + } + + @Test + public void testIsNotBinaryTypeWithCss() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(org.springframework.http.MediaType.valueOf("text/css")); + assertFalse(RecordUtils.isNotBinaryType(headers)); + } + + @Test + public void testIsNotBinaryTypeWithNullHeaders() { + assertTrue(RecordUtils.isNotBinaryType(null)); + } + + @Test + public void testIsNotBinaryTypeWithNoContentType() { + HttpHeaders headers = new HttpHeaders(); + assertTrue(RecordUtils.isNotBinaryType(headers)); + } + + @Test + public void testGetHeadersWithSingleValue() { + HttpHeaders headers = new HttpHeaders(); + headers.set("Content-Type", "application/json"); + Map result = RecordUtils.getHeaders(headers); + assertEquals("application/json", result.get("Content-Type")); + } + + @Test + public void testGetHeadersWithMultipleValues() { + HttpHeaders headers = new HttpHeaders(); + headers.add("Accept", "application/json"); + headers.add("Accept", "text/html"); + Map result = RecordUtils.getHeaders(headers); + assertEquals("application/json,text/html", result.get("Accept")); + } + + @Test + public void testGetHeadersWithNullHeaders() { + Map result = RecordUtils.getHeaders(null); + assertTrue(result.isEmpty()); + } + + @Test + public void testGetHeadersWithEmptyHeaders() { + HttpHeaders headers = new HttpHeaders(); + Map result = RecordUtils.getHeaders(headers); + assertTrue(result.isEmpty()); + } +} \ No newline at end of file diff --git a/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/pom.xml b/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/pom.xml index e5192d9829c4..86928b4f048a 100644 --- a/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/pom.xml +++ b/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/pom.xml @@ -78,5 +78,6 @@ shenyu-spring-boot-starter-plugin-ai-request-transformer shenyu-spring-boot-starter-plugin-ai-response-transformer shenyu-spring-boot-starter-plugin-mcp-server + shenyu-spring-boot-starter-plugin-http-record diff --git a/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/pom.xml b/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/pom.xml new file mode 100644 index 000000000000..f1ae64758f57 --- /dev/null +++ b/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/pom.xml @@ -0,0 +1,36 @@ + + + + + + org.apache.shenyu + shenyu-spring-boot-starter-plugin + 2.7.2-SNAPSHOT + + 4.0.0 + shenyu-spring-boot-starter-plugin-http-record + + + + org.apache.shenyu + shenyu-plugin-record + ${project.version} + + + + diff --git a/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/src/main/java/org/apache/shenyu/springboot/plugin/http/record/HttpRecordPluginConfiguration.java b/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/src/main/java/org/apache/shenyu/springboot/plugin/http/record/HttpRecordPluginConfiguration.java new file mode 100644 index 000000000000..403fadc29aa7 --- /dev/null +++ b/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/src/main/java/org/apache/shenyu/springboot/plugin/http/record/HttpRecordPluginConfiguration.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.shenyu.springboot.plugin.http.record; + +import jakarta.annotation.PreDestroy; +import org.apache.shenyu.plugin.record.HttpRecordPlugin; +import org.apache.shenyu.plugin.record.collector.HttpRecordCollector; +import org.apache.shenyu.plugin.record.config.ShenyuHttpRecordCenterConfig; +import org.apache.shenyu.plugin.record.handler.HttpRecordPluginDataHandler; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * HttpRecordPluginConfiguration. + * + *

Spring Boot auto-configuration for the http record plugin. + * Enabled by default; can be disabled via {@code shenyu.plugins.http-record.enabled=false}.

+ */ +@Configuration +@ConditionalOnProperty(value = "shenyu.plugins.http-record.enabled", havingValue = "true", matchIfMissing = true) +public class HttpRecordPluginConfiguration { + + /** + * Http record plugin bean. + * + * @return the http record plugin + */ + @Bean + @ConditionalOnMissingBean + public HttpRecordPlugin httpRecordPlugin() { + return new HttpRecordPlugin(); + } + + /** + * Http record plugin data handler bean. + * + * @return the data handler + */ + @Bean + public HttpRecordPluginDataHandler httpRecordPluginDataHandler() { + return new HttpRecordPluginDataHandler(); + } + + /** + * Shenyu http record center config bean. + * + * @return the config bean + */ + @Bean + @ConfigurationProperties(prefix = "shenyu.http-record") + ShenyuHttpRecordCenterConfig shenyuRecordCenterConfig() { + return new ShenyuHttpRecordCenterConfig(); + } + + /** + * Cleanup on application shutdown. + */ + @PreDestroy + public void destroy() { + HttpRecordCollector.INSTANCE.stop(); + } +} diff --git a/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/src/main/resources/META-INF/spring.factories b/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/src/main/resources/META-INF/spring.factories new file mode 100644 index 000000000000..1ac3b2db7f95 --- /dev/null +++ b/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +org.apache.shenyu.springboot.plugin.http.record.HttpRecordPluginConfiguration diff --git a/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/src/main/resources/META-INF/spring.provides b/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/src/main/resources/META-INF/spring.provides new file mode 100644 index 000000000000..3fb09e4a7991 --- /dev/null +++ b/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/src/main/resources/META-INF/spring.provides @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +provides: shenyu-spring-boot-starter-plugin-http-record diff --git a/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 000000000000..95d94f4a7792 --- /dev/null +++ b/shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/shenyu-spring-boot-starter-plugin-http-record/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.shenyu.springboot.plugin.http.record.HttpRecordPluginConfiguration