diff --git a/poetry.lock b/poetry.lock index c401958..9e0e263 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "backoff" @@ -555,13 +555,13 @@ files = [ [[package]] name = "pysquril" -version = "0.6.2" +version = "0.7.0" description = "Python implementation of structured URI query language" optional = false python-versions = "<4.0,>=3.7" files = [ - {file = "pysquril-0.6.2-py3-none-any.whl", hash = "sha256:bffb62ab8fe09cf9ab48251b1d1b216b245e57660da5208bb39d9ff9b5af38db"}, - {file = "pysquril-0.6.2.tar.gz", hash = "sha256:20ce349803624443473c044b6369e3d112fcc036db542159144157d5b9be2ca8"}, + {file = "pysquril-0.7.0-py3-none-any.whl", hash = "sha256:07c55ada548cc3fe98aba14976b605c9f37c0fad196c90a4a33750b5a6f64d5c"}, + {file = "pysquril-0.7.0.tar.gz", hash = "sha256:d892d808d71a73c908864d0291b841444a3156ff75dab0cb56a9ae93bb62d601"}, ] [package.dependencies] @@ -603,7 +603,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -736,7 +735,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\")"} [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] diff --git a/tsdfileapi/api.py b/tsdfileapi/api.py index 27e5426..dc30b8e 100644 --- a/tsdfileapi/api.py +++ b/tsdfileapi/api.py @@ -975,7 +975,6 @@ def initialize(self, backend: str, namespace: str, endpoint: str) -> None: self.allow_info = self.backend_config["allow_info"] self.allow_delete = self.backend_config["allow_delete"] self.allow_rpc = self.backend_config.get("allow_rpc") - self.export_max = self.backend_config["export_max_num_list"] self.has_posix_ownership = self.backend_config["has_posix_ownership"] self.check_tenant = self.backend_config.get("check_tenant") self.mq_config = self.backend_config.get("mq") @@ -1609,10 +1608,6 @@ def list_files(self, path: str, tenant: str, root: str) -> None: next_page = None baseuri = self._base_uri() nextref = f"{baseuri}?page={next_page}" if next_page else None - if self.export_max and len(files) > self.export_max: - raise ClientError( - f"number of files exceed configured maximum: {self.export_max}" - ) names = [] times = [] exportable = [] diff --git a/tsdfileapi/config/config-test-container-postgres.yaml b/tsdfileapi/config/config-test-container-postgres.yaml index 182d4f3..e0d04a5 100644 --- a/tsdfileapi/config/config-test-container-postgres.yaml +++ b/tsdfileapi/config/config-test-container-postgres.yaml @@ -53,7 +53,6 @@ backends: allow_info: true allow_list: true allow_rpc: true - export_max_num_list: null export_path: /tmp/pXX/apps export_policy: default: @@ -75,7 +74,6 @@ backends: allow_export: true allow_info: true allow_list: true - export_max_num_list: null export_path: /file-api/tsdfileapi/data/tsd/pXX/export export_policy: default: @@ -94,7 +92,6 @@ backends: allow_export: true allow_info: true allow_list: true - export_max_num_list: null export_path: /tmp/pXX/import export_policy: default: @@ -124,7 +121,6 @@ backends: allow_export: true allow_info: true allow_list: true - export_max_num_list: null export_path: /tmp/pXX export_policy: default: @@ -151,7 +147,6 @@ backends: allow_info: true allow_list: true allow_rpc: true - export_max_num_list: null export_path: /tmp/pXX/survey export_policy: default: @@ -177,7 +172,6 @@ debug: true default_file_owner: pXX-nobody disallowed_start_chars: '~' export_chunk_size: 512000 -export_max_num_list: 100 jwt_secret: null jwt_test_secret: jS25aQbePizfTsetg8LbFsNKl1W6wi4nQaBj705ofWA= log_level: info diff --git a/tsdfileapi/config/config-test.yaml b/tsdfileapi/config/config-test.yaml index c018052..0390de6 100644 --- a/tsdfileapi/config/config-test.yaml +++ b/tsdfileapi/config/config-test.yaml @@ -10,7 +10,6 @@ valid_tenant_regex: ^[0-9a-z]+$ tenant_string_pattern: pXX allowed_symlinks: - "/tsd" -export_max_num_list: 100 export_chunk_size: 512000 max_body_size: 5368709120 default_file_owner: pXX-nobody @@ -44,7 +43,6 @@ backends: ensure_tenant_in_group_name: false has_posix_ownership: false import_path: /tmp/pXX - export_max_num_list: None export_path: /tmp/pXX/export apps_files: allow_list: true @@ -63,7 +61,6 @@ backends: enabled: false has_posix_ownership: false import_path: /tmp/pXX - export_max_num_list: None export_path: /tmp/pXX/export files_export: allow_list: true @@ -82,7 +79,6 @@ backends: enabled: false has_posix_ownership: false import_path: /tmp/pXX - export_max_num_list: None export_path: /tmp/pXX/export cluster: allow_list: true @@ -106,7 +102,6 @@ backends: ensure_tenant_in_group_name: false has_posix_ownership: false import_path: /tmp/pXX/cluster/ - export_max_num_list: None export_path: /tmp/pXX/export survey: allow_list: true @@ -128,7 +123,6 @@ backends: ensure_tenant_in_group_name: false has_posix_ownership: false import_path: /tmp/pXX/survey - export_max_num_list: None export_path: /tmp/pXX/export backup_deletes: backup_days: 90 @@ -162,7 +156,6 @@ backends: enabled: false has_posix_ownership: false import_path: /tmp/pXX/store - export_max_num_list: None export_path: /tmp/pXX/store dbs: survey: diff --git a/tsdfileapi/config/config.yaml b/tsdfileapi/config/config.yaml index 848c28a..897889c 100644 --- a/tsdfileapi/config/config.yaml +++ b/tsdfileapi/config/config.yaml @@ -8,7 +8,6 @@ requestor_claim_name: user tenant_claim_name: proj valid_tenant_regex: ^[0-9a-z]+$ tenant_string_pattern: pXX -export_max_num_list: 100 export_chunk_size: 512000 max_body_size: 5368709120 default_file_owner: pXX-nobody @@ -17,4 +16,4 @@ jwt_test_secret: jS25aQbePizfTsetg8LbFsNKl1W6wi4nQaBj705ofWA= jwt_secret: None nacl_public: {'public': 'mZQEzkyi7bCvmDVfHGsU/7HX1+gT/R3PkSnyDU4OaiY=', 'private': 'fTEB1MZz8MskkZHSIM9ypxJc4e45Z8fmLGGXkUrp1hQ='} log_level: info -backends: {'disk': {'store': {'has_posix_ownership': False, 'export_max_num_list': None, 'import_path': '/tmp/pXX', 'export_path': '/tmp/pXX', 'allow_export': True, 'allow_list': True, 'allow_info': True, 'allow_delete': True, 'export_policy': {'default': {'enabled': False}}, 'group_logic': {'default_url_group': None, 'default_memberships': ['pXX-member-group'], 'enabled': False}, 'request_hook': {'enabled': False}}, 'apps_files': {'has_posix_ownership': False, 'export_max_num_list': None, 'import_path': '/tmp/pXX', 'export_path': '/tmp/pXX', 'allow_export': True, 'allow_list': True, 'allow_info': True, 'allow_delete': True, 'export_policy': {'default': {'enabled': False}}, 'group_logic': {'default_url_group': None, 'default_memberships': ['pXX-member-group'], 'enabled': False}, 'request_hook': {'enabled': False}}}, 'dbs': {'apps_tables': {'db': {'engine': 'sqlite', 'path': '/tmp/pXX', 'table_structure': None, 'mq': None}}}} +backends: {'disk': {'store': {'has_posix_ownership': False, 'import_path': '/tmp/pXX', 'export_path': '/tmp/pXX', 'allow_export': True, 'allow_list': True, 'allow_info': True, 'allow_delete': True, 'export_policy': {'default': {'enabled': False}}, 'group_logic': {'default_url_group': None, 'default_memberships': ['pXX-member-group'], 'enabled': False}, 'request_hook': {'enabled': False}}, 'apps_files': {'has_posix_ownership': False, 'import_path': '/tmp/pXX', 'export_path': '/tmp/pXX', 'allow_export': True, 'allow_list': True, 'allow_info': True, 'allow_delete': True, 'export_policy': {'default': {'enabled': False}}, 'group_logic': {'default_url_group': None, 'default_memberships': ['pXX-member-group'], 'enabled': False}, 'request_hook': {'enabled': False}}}, 'dbs': {'apps_tables': {'db': {'engine': 'sqlite', 'path': '/tmp/pXX', 'table_structure': None, 'mq': None}}}} diff --git a/tsdfileapi/config/example-file-api-config.yaml b/tsdfileapi/config/example-file-api-config.yaml index c396762..d19f56f 100644 --- a/tsdfileapi/config/example-file-api-config.yaml +++ b/tsdfileapi/config/example-file-api-config.yaml @@ -10,7 +10,6 @@ requestor_claim_name: 'user' tenant_claim_name: 'proj' valid_tenant_regex: '^[0-9a-z]+$' tenant_string_pattern: 'pXX' -export_max_num_list: 100 export_chunk_size: 512000 backup_days: 90 # endpoint backends diff --git a/tsdfileapi/defaults.py b/tsdfileapi/defaults.py index b3826b9..dc0ad7d 100644 --- a/tsdfileapi/defaults.py +++ b/tsdfileapi/defaults.py @@ -24,7 +24,6 @@ "tenant_claim_name": "proj", "valid_tenant_regex": "^[0-9a-z]+$", "tenant_string_pattern": "pXX", - "export_max_num_list": 100, "export_chunk_size": 512000, "max_body_size": 5368709120, "default_file_owner": "pXX-nobody", @@ -59,7 +58,6 @@ "disk": { "publication": { "has_posix_ownership": False, - "export_max_num_list": None, "import_path": f"{TMP}/pXX", "export_path": f"{TMP}/pXX", "allow_export": True, @@ -80,7 +78,6 @@ }, "apps_files": { "has_posix_ownership": False, - "export_max_num_list": None, "import_path": f"{TMP}/pXX/apps", "export_path": f"{TMP}/pXX/apps", "allow_export": True, @@ -106,7 +103,6 @@ }, "survey": { "has_posix_ownership": False, - "export_max_num_list": None, "import_path": f"{TMP}/pXX/survey", "export_path": f"{TMP}/pXX/survey", "allow_export": True, @@ -144,7 +140,6 @@ }, "files_import": { "has_posix_ownership": True, - "export_max_num_list": None, "import_path": f"{TMP}/pXX/import", "export_path": f"{TMP}/pXX/import", "allow_export": True, @@ -170,7 +165,6 @@ }, "files_export": { "has_posix_ownership": False, - "export_max_num_list": None, "import_path": f"{TMP}/pXX", "export_path": f"{os.getcwd()}/tsdfileapi/data/tsd/pXX/export", "allow_export": True,