From f48c46c9ace447c02c56c9243ea3fcfc95ca3487 Mon Sep 17 00:00:00 2001 From: Joakim Melseth Date: Tue, 28 Jul 2026 09:02:44 +0000 Subject: [PATCH 1/3] fix: [DHIS2-21855] limit concurrent api requests --- engine/package.json | 3 ++ engine/src/links/RestAPILink.ts | 7 +++- examples/cra/yarn.lock | 46 ++++++++++++---------- examples/query-playground/yarn.lock | 59 ++++++++++++++++++----------- yarn.lock | 12 ++++++ 5 files changed, 84 insertions(+), 43 deletions(-) diff --git a/engine/package.json b/engine/package.json index 6ba147094..2c35b9a7b 100644 --- a/engine/package.json +++ b/engine/package.json @@ -32,5 +32,8 @@ "type-check:watch": "yarn type-check --watch", "test": "d2-app-scripts test", "coverage": "yarn test --coverage" + }, + "dependencies": { + "p-limit": "^7.3.1" } } diff --git a/engine/src/links/RestAPILink.ts b/engine/src/links/RestAPILink.ts index c452ee327..dfff50405 100644 --- a/engine/src/links/RestAPILink.ts +++ b/engine/src/links/RestAPILink.ts @@ -1,3 +1,4 @@ +import plimit from 'p-limit' import { LRUCache } from '../helpers/LRUCache' import type { DataEngineConfig } from '../types/DataEngineConfig' import type { @@ -13,6 +14,8 @@ import { joinPath } from './RestAPILink/path' import { queryToRequestOptions } from './RestAPILink/queryToRequestOptions' import { queryToResourcePath } from './RestAPILink/queryToResourcePath' +const limit = plimit(100) + export class RestAPILink implements DataEngineLink { public readonly config: DataEngineConfig public readonly versionedApiPath: string @@ -30,7 +33,9 @@ export class RestAPILink implements DataEngineLink { } private fetch(path: string, options: RequestInit): Promise { - return fetchData(joinPath(this.config.baseUrl, path), options, this) + return limit(() => + fetchData(joinPath(this.config.baseUrl, path), options, this) + ) } public executeResourceQuery( diff --git a/examples/cra/yarn.lock b/examples/cra/yarn.lock index f37b1dbc8..0cfaadb31 100644 --- a/examples/cra/yarn.lock +++ b/examples/cra/yarn.lock @@ -1047,49 +1047,55 @@ integrity sha512-ij4wRiunFfaJxjB0BdrYHIH8FxBJpOwNPhhAcunlmPdXudL1WQV1qoP9un6JsEBAgQH+7UXyyjh0g7jTxXK6tg== "@dhis2/app-runtime@file:../../runtime": - version "3.17.0-beta.2" - dependencies: - "@dhis2/app-service-alerts" "3.17.0-beta.2" - "@dhis2/app-service-config" "3.17.0-beta.2" - "@dhis2/app-service-data" "3.17.0-beta.2" - "@dhis2/app-service-offline" "3.17.0-beta.2" - "@dhis2/app-service-plugin" "3.17.0-beta.2" - "@dhis2/app-service-user" "3.17.0-beta.2" + version "3.17.3" + dependencies: + "@dhis2/app-service-alerts" "3.17.3" + "@dhis2/app-service-config" "3.17.3" + "@dhis2/app-service-data" "3.17.3" + "@dhis2/app-service-offline" "3.17.3" + "@dhis2/app-service-plugin" "3.17.3" + "@dhis2/app-service-user" "3.17.3" prop-types "^15.7.2" -"@dhis2/app-service-alerts@3.17.0-beta.2", "@dhis2/app-service-alerts@file:../../services/alerts": - version "3.17.0-beta.2" +"@dhis2/app-service-alerts@3.17.3", "@dhis2/app-service-alerts@file:../../services/alerts": + version "3.17.3" dependencies: prop-types "^15.7.2" -"@dhis2/app-service-config@3.17.0-beta.2", "@dhis2/app-service-config@file:../../services/config": - version "3.17.0-beta.2" +"@dhis2/app-service-config@3.17.3", "@dhis2/app-service-config@file:../../services/config": + version "3.17.3" dependencies: prop-types "^15.7.2" -"@dhis2/app-service-data@3.17.0-beta.2", "@dhis2/app-service-data@file:../../services/data": - version "3.17.0-beta.2" +"@dhis2/app-service-data@3.17.3", "@dhis2/app-service-data@file:../../services/data": + version "3.17.3" dependencies: + "@dhis2/data-engine" "3.17.3" "@tanstack/react-query" "^4.36.1" prop-types "^15.7.2" -"@dhis2/app-service-offline@3.17.0-beta.2", "@dhis2/app-service-offline@file:../../services/offline": - version "3.17.0-beta.2" +"@dhis2/app-service-offline@3.17.3", "@dhis2/app-service-offline@file:../../services/offline": + version "3.17.3" dependencies: lodash "^4.17.21" prop-types "^15.7.2" -"@dhis2/app-service-plugin@3.17.0-beta.2", "@dhis2/app-service-plugin@file:../../services/plugin": - version "3.17.0-beta.2" +"@dhis2/app-service-plugin@3.17.3", "@dhis2/app-service-plugin@file:../../services/plugin": + version "3.17.3" dependencies: post-robot "^10.0.46" prop-types "^15.7.2" -"@dhis2/app-service-user@3.17.0-beta.2", "@dhis2/app-service-user@file:../../services/user": - version "3.17.0-beta.2" +"@dhis2/app-service-user@3.17.3", "@dhis2/app-service-user@file:../../services/user": + version "3.17.3" dependencies: prop-types "^15.7.2" +"@dhis2/data-engine@3.17.3": + version "3.17.3" + resolved "https://registry.yarnpkg.com/@dhis2/data-engine/-/data-engine-3.17.3.tgz#0347416e9919efbf4d9739c4141fa543f89669ad" + integrity sha512-hLXt7LFrFitR7QgKfGQ3ComTLrY5IAdtERonhdo/SIrsRYWoeVaMiCOkUUzC48pEaeo1/BL5qwA7Tw7jZgROQw== + "@hapi/address@2.x.x": version "2.1.4" resolved "https://registry.yarnpkg.com/@hapi/address/-/address-2.1.4.tgz#5d67ed43f3fd41a69d4b9ff7b56e7c0d1d0a81e5" diff --git a/examples/query-playground/yarn.lock b/examples/query-playground/yarn.lock index ab1cc383e..1e5a6cbd3 100644 --- a/examples/query-playground/yarn.lock +++ b/examples/query-playground/yarn.lock @@ -1715,46 +1715,47 @@ prop-types "^15.7.2" "@dhis2/app-runtime@^3.12.0", "@dhis2/app-runtime@file:../../runtime": - version "3.17.0-beta.2" - dependencies: - "@dhis2/app-service-alerts" "3.17.0-beta.2" - "@dhis2/app-service-config" "3.17.0-beta.2" - "@dhis2/app-service-data" "3.17.0-beta.2" - "@dhis2/app-service-offline" "3.17.0-beta.2" - "@dhis2/app-service-plugin" "3.17.0-beta.2" - "@dhis2/app-service-user" "3.17.0-beta.2" + version "3.17.3" + dependencies: + "@dhis2/app-service-alerts" "3.17.3" + "@dhis2/app-service-config" "3.17.3" + "@dhis2/app-service-data" "3.17.3" + "@dhis2/app-service-offline" "3.17.3" + "@dhis2/app-service-plugin" "3.17.3" + "@dhis2/app-service-user" "3.17.3" prop-types "^15.7.2" -"@dhis2/app-service-alerts@3.16.0", "@dhis2/app-service-alerts@3.17.0-beta.2", "@dhis2/app-service-alerts@file:../../services/alerts": - version "3.17.0-beta.2" +"@dhis2/app-service-alerts@3.16.0", "@dhis2/app-service-alerts@3.17.3", "@dhis2/app-service-alerts@file:../../services/alerts": + version "3.17.3" dependencies: prop-types "^15.7.2" -"@dhis2/app-service-config@3.16.0", "@dhis2/app-service-config@3.17.0-beta.2", "@dhis2/app-service-config@file:../../services/config": - version "3.17.0-beta.2" +"@dhis2/app-service-config@3.16.0", "@dhis2/app-service-config@3.17.3", "@dhis2/app-service-config@file:../../services/config": + version "3.17.3" dependencies: prop-types "^15.7.2" -"@dhis2/app-service-data@3.16.0", "@dhis2/app-service-data@3.17.0-beta.2", "@dhis2/app-service-data@file:../../services/data": - version "3.17.0-beta.2" +"@dhis2/app-service-data@3.16.0", "@dhis2/app-service-data@3.17.3", "@dhis2/app-service-data@file:../../services/data": + version "3.17.3" dependencies: + "@dhis2/data-engine" "3.17.3" "@tanstack/react-query" "^4.36.1" prop-types "^15.7.2" -"@dhis2/app-service-offline@3.16.0", "@dhis2/app-service-offline@3.17.0-beta.2", "@dhis2/app-service-offline@file:../../services/offline": - version "3.17.0-beta.2" +"@dhis2/app-service-offline@3.16.0", "@dhis2/app-service-offline@3.17.3", "@dhis2/app-service-offline@file:../../services/offline": + version "3.17.3" dependencies: lodash "^4.17.21" prop-types "^15.7.2" -"@dhis2/app-service-plugin@3.16.0", "@dhis2/app-service-plugin@3.17.0-beta.2", "@dhis2/app-service-plugin@file:../../services/plugin": - version "3.17.0-beta.2" +"@dhis2/app-service-plugin@3.16.0", "@dhis2/app-service-plugin@3.17.3", "@dhis2/app-service-plugin@file:../../services/plugin": + version "3.17.3" dependencies: post-robot "^10.0.46" prop-types "^15.7.2" -"@dhis2/app-service-user@3.16.0", "@dhis2/app-service-user@3.17.0-beta.2", "@dhis2/app-service-user@file:../../services/user": - version "3.17.0-beta.2" +"@dhis2/app-service-user@3.16.0", "@dhis2/app-service-user@3.17.3", "@dhis2/app-service-user@file:../../services/user": + version "3.17.3" dependencies: prop-types "^15.7.2" @@ -1862,8 +1863,10 @@ i18next "^10.3" moment "^2.24.0" -"@dhis2/data-engine@file:../../engine": - version "3.17.0-beta.2" +"@dhis2/data-engine@3.17.3", "@dhis2/data-engine@file:../../engine": + version "3.17.3" + dependencies: + p-limit "^7.3.1" "@dhis2/multi-calendar-dates@2.0.0": version "2.0.0" @@ -7725,6 +7728,13 @@ p-limit@^3.0.2: dependencies: yocto-queue "^0.1.0" +p-limit@^7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-7.3.1.tgz#ded48cbfa10b161a9928120261fa82c9a282eb3f" + integrity sha512-0trZaiG7Y7kN/Egy9a8j47t9osC0Tch4PaIWd9yGF6bvmlk7muExRvGNYb8sXBwEKMoNKsbNN9P8EefuQekE4Q== + dependencies: + yocto-queue "^1.2.1" + p-locate@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4" @@ -10688,6 +10698,11 @@ yocto-queue@^0.1.0: resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== +yocto-queue@^1.2.1: + version "1.2.2" + resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.2.2.tgz#3e09c95d3f1aa89a58c114c99223edf639152c00" + integrity sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ== + zalgo-promise@^1, zalgo-promise@^1.0.11, zalgo-promise@^1.0.3: version "1.0.48" resolved "https://registry.yarnpkg.com/zalgo-promise/-/zalgo-promise-1.0.48.tgz#9e33eef502d5ed9f5a09fc5728c833c3e87afa2e" diff --git a/yarn.lock b/yarn.lock index 32ea8d9e9..29490acb4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8944,6 +8944,13 @@ p-limit@^3.0.2: dependencies: yocto-queue "^0.1.0" +p-limit@^7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-7.3.1.tgz#ded48cbfa10b161a9928120261fa82c9a282eb3f" + integrity sha512-0trZaiG7Y7kN/Egy9a8j47t9osC0Tch4PaIWd9yGF6bvmlk7muExRvGNYb8sXBwEKMoNKsbNN9P8EefuQekE4Q== + dependencies: + yocto-queue "^1.2.1" + p-locate@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz" @@ -12177,6 +12184,11 @@ yocto-queue@^0.1.0: resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== +yocto-queue@^1.2.1: + version "1.2.2" + resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.2.2.tgz#3e09c95d3f1aa89a58c114c99223edf639152c00" + integrity sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ== + zalgo-promise@^1, zalgo-promise@^1.0.11, zalgo-promise@^1.0.3: version "1.0.48" resolved "https://registry.npmjs.org/zalgo-promise/-/zalgo-promise-1.0.48.tgz" From d76cd4874527c78736eb421b25bdbc09254de2ff Mon Sep 17 00:00:00 2001 From: Joakim Melseth Date: Wed, 29 Jul 2026 07:19:17 +0000 Subject: [PATCH 2/3] test: fix tests and add concurrent test --- engine/jest.config.js | 5 +++++ engine/src/links/RestAPILink.test.ts | 33 +++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 engine/jest.config.js diff --git a/engine/jest.config.js b/engine/jest.config.js new file mode 100644 index 000000000..e8738d5eb --- /dev/null +++ b/engine/jest.config.js @@ -0,0 +1,5 @@ +module.exports = { + transformIgnorePatterns: [ + '/node_modules/(?!(moment/dist|p-limit|yocto-queue)/)', + ], +} diff --git a/engine/src/links/RestAPILink.test.ts b/engine/src/links/RestAPILink.test.ts index b784e5f77..4a82c2571 100644 --- a/engine/src/links/RestAPILink.test.ts +++ b/engine/src/links/RestAPILink.test.ts @@ -2,11 +2,16 @@ import { fetchData } from './RestAPILink/fetchData' import { RestAPILink } from '.' jest.mock('./RestAPILink/fetchData', () => ({ - fetchData: jest.fn(async () => null), + fetchData: jest.fn(), })) +const mockedFetchData = fetchData as jest.MockedFunction + describe('RestAPILink', () => { + beforeEach(() => mockedFetchData.mockReset()) + it('should call fetch with the expected URL', async () => { + mockedFetchData.mockResolvedValue(null) const link = new RestAPILink({ baseUrl: 'http://url', apiVersion: 42 }) await link.executeResourceQuery('read', { resource: 'something' }, {}) expect(fetchData).toHaveBeenCalledWith( @@ -23,4 +28,30 @@ describe('RestAPILink', () => { }) ) }) + it('should limit the number of concurrent requests to 100', async () => { + const resolvers: Array<() => void> = [] + mockedFetchData.mockImplementation( + () => + new Promise((resolve) => { + resolvers.push(() => resolve(null)) + }) + ) + + const link = new RestAPILink({ baseUrl: 'http://url', apiVersion: 42 }) + Array.from({ length: 101 }, () => + link.executeResourceQuery('read', { resource: 'something' }, {}) + ) + + const flushPromises = () => + new Promise((resolve) => setTimeout(resolve, 0)) + + await flushPromises() // flush microtask queue so p-limit dispatches the 100 concurrent calls + + expect(fetchData).toHaveBeenCalledTimes(100) + + resolvers[0]() // resolve the first call, freeing a slot for the 101st queued request + await flushPromises() // wait for p-limit to dispatch the 101st call + + expect(fetchData).toHaveBeenCalledTimes(101) + }) }) From e800a7d069cfe93040411c44f2a0e6ce418e3a47 Mon Sep 17 00:00:00 2001 From: Joakim Melseth Date: Wed, 29 Jul 2026 13:03:28 +0000 Subject: [PATCH 3/3] test: ignore pattern for service data --- services/data/jest.config.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/data/jest.config.js b/services/data/jest.config.js index d1fe647fd..06a7d80ec 100644 --- a/services/data/jest.config.js +++ b/services/data/jest.config.js @@ -1,4 +1,8 @@ module.exports = { + transformIgnorePatterns: [ + '/node_modules/(?!(moment/dist|p-limit|yocto-queue)/)', + ], + collectCoverageFrom: [ 'src/**/*.(js|jsx|ts|tsx)', '!src/index.ts',