From 698ec83c96f8429e187c9dc1dde01420e0f6dc36 Mon Sep 17 00:00:00 2001 From: zman Date: Wed, 27 Aug 2025 12:55:29 -0400 Subject: [PATCH] first commit --- .vscode/settings.json | 1 + f/app_custom/folder.meta.yaml | 5 +++ f/app_groups/folder.meta.yaml | 5 +++ f/app_themes/folder.meta.yaml | 5 +++ f/app_themes/theme_0.resource.yaml | 5 +++ u/joshuakrzemien/db_test.py | 15 +++++++++ u/joshuakrzemien/db_test.script.lock | 13 ++++++++ u/joshuakrzemien/db_test.script.yaml | 9 +++++ u/joshuakrzemien/mtg_json_download.py | 33 +++++++++++++++++++ .../mtg_json_download.script.lock | 13 ++++++++ .../mtg_json_download.script.yaml | 10 ++++++ u/joshuakrzemien/mtgjson_flow.flow/flow.yaml | 26 +++++++++++++++ u/joshuakrzemien/mtgjson_unzip.py | 22 +++++++++++++ u/joshuakrzemien/mtgjson_unzip.script.lock | 10 ++++++ u/joshuakrzemien/mtgjson_unzip.script.yaml | 14 ++++++++ .../slick_postgresql.resource.yaml | 10 ++++++ .../slick_postgresql.variable.yaml | 3 ++ wmill-lock.yaml | 5 +++ 18 files changed, 204 insertions(+) create mode 100644 .vscode/settings.json create mode 100644 f/app_custom/folder.meta.yaml create mode 100644 f/app_groups/folder.meta.yaml create mode 100644 f/app_themes/folder.meta.yaml create mode 100644 f/app_themes/theme_0.resource.yaml create mode 100644 u/joshuakrzemien/db_test.py create mode 100644 u/joshuakrzemien/db_test.script.lock create mode 100644 u/joshuakrzemien/db_test.script.yaml create mode 100644 u/joshuakrzemien/mtg_json_download.py create mode 100644 u/joshuakrzemien/mtg_json_download.script.lock create mode 100644 u/joshuakrzemien/mtg_json_download.script.yaml create mode 100644 u/joshuakrzemien/mtgjson_flow.flow/flow.yaml create mode 100644 u/joshuakrzemien/mtgjson_unzip.py create mode 100644 u/joshuakrzemien/mtgjson_unzip.script.lock create mode 100644 u/joshuakrzemien/mtgjson_unzip.script.yaml create mode 100644 u/joshuakrzemien/slick_postgresql.resource.yaml create mode 100644 u/joshuakrzemien/slick_postgresql.variable.yaml create mode 100644 wmill-lock.yaml diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/f/app_custom/folder.meta.yaml b/f/app_custom/folder.meta.yaml new file mode 100644 index 0000000..573f64a --- /dev/null +++ b/f/app_custom/folder.meta.yaml @@ -0,0 +1,5 @@ +summary: null +display_name: App Custom Components +extra_perms: + g/all: false +owners: [] diff --git a/f/app_groups/folder.meta.yaml b/f/app_groups/folder.meta.yaml new file mode 100644 index 0000000..d224ebe --- /dev/null +++ b/f/app_groups/folder.meta.yaml @@ -0,0 +1,5 @@ +summary: null +display_name: App Groups +extra_perms: + g/all: false +owners: [] diff --git a/f/app_themes/folder.meta.yaml b/f/app_themes/folder.meta.yaml new file mode 100644 index 0000000..7c672ae --- /dev/null +++ b/f/app_themes/folder.meta.yaml @@ -0,0 +1,5 @@ +summary: null +display_name: App Themes +extra_perms: + g/all: false +owners: [] diff --git a/f/app_themes/theme_0.resource.yaml b/f/app_themes/theme_0.resource.yaml new file mode 100644 index 0000000..c9f2984 --- /dev/null +++ b/f/app_themes/theme_0.resource.yaml @@ -0,0 +1,5 @@ +description: The default app theme +value: + name: Default Theme + value: '' +resource_type: app_theme diff --git a/u/joshuakrzemien/db_test.py b/u/joshuakrzemien/db_test.py new file mode 100644 index 0000000..7d9283a --- /dev/null +++ b/u/joshuakrzemien/db_test.py @@ -0,0 +1,15 @@ +import os +import wmill +from sqlalchemy import create_engine +import psycopg2 + +# You can import any PyPi package. +# See here for more info: https://www.windmill.dev/docs/advanced/dependencies_in_python + +# you can use typed resources by doing a type alias to dict +#postgresql = dict + +def main(): + db = wmill.client.get_resource('u/joshuakrzemien/slick_postgresql') + engine = create_engine(f"postgresql+psycopg2://postgres:{db['password']}@192.168.1.41:5431/windmill_test_db") + engine.connect() \ No newline at end of file diff --git a/u/joshuakrzemien/db_test.script.lock b/u/joshuakrzemien/db_test.script.lock new file mode 100644 index 0000000..605e755 --- /dev/null +++ b/u/joshuakrzemien/db_test.script.lock @@ -0,0 +1,13 @@ +# py: 3.11 +anyio==4.10.0 +certifi==2025.8.3 +greenlet==3.2.4 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.10 +psycopg2-binary==2.9.10 +sniffio==1.3.1 +sqlalchemy==2.0.43 +typing-extensions==4.15.0 +wmill==1.536.0 \ No newline at end of file diff --git a/u/joshuakrzemien/db_test.script.yaml b/u/joshuakrzemien/db_test.script.yaml new file mode 100644 index 0000000..ebb8499 --- /dev/null +++ b/u/joshuakrzemien/db_test.script.yaml @@ -0,0 +1,9 @@ +summary: db test +description: '' +lock: '!inline u/joshuakrzemien/db_test.script.lock' +kind: script +schema: + $schema: 'https://json-schema.org/draft/2020-12/schema' + type: object + properties: {} + required: [] diff --git a/u/joshuakrzemien/mtg_json_download.py b/u/joshuakrzemien/mtg_json_download.py new file mode 100644 index 0000000..8bb9181 --- /dev/null +++ b/u/joshuakrzemien/mtg_json_download.py @@ -0,0 +1,33 @@ +import os +import wmill +import requests + +# You can import any PyPi package. +# See here for more info: https://www.windmill.dev/docs/advanced/dependencies_in_python + +# you can use typed resources by doing a type alias to dict +#postgresql = dict + +def main(): + url = "https://mtgjson.com/api/v5/TcgplayerSkus.json.zip" + files = { + "skus": "https://mtgjson.com/api/v5/TcgplayerSkus.json.zip", + "identifiers": "https://mtgjson.com/api/v5/AllIdentifiers.json.zip" + } + success = {} + for file_type, url in files.items(): + response = requests.get(url) + response.raise_for_status() + file_name = url.split("/")[-1] + content_type = response.headers.get("Content-Type") + if content_type == 'application/zip': + # save file + path = f"./shared/{file_name}" + with open(path, 'wb') as f: + f.write(response.content) + success[file_name] = True + else: + success[file_name] = False + return success + # success {"TcgplayerSkus.json.zip": True, "AllIdentifiers.json.zip": True} + diff --git a/u/joshuakrzemien/mtg_json_download.script.lock b/u/joshuakrzemien/mtg_json_download.script.lock new file mode 100644 index 0000000..ad26b7a --- /dev/null +++ b/u/joshuakrzemien/mtg_json_download.script.lock @@ -0,0 +1,13 @@ +# py: 3.11 +anyio==4.10.0 +certifi==2025.8.3 +charset-normalizer==3.4.3 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.10 +requests==2.32.5 +sniffio==1.3.1 +typing-extensions==4.15.0 +urllib3==2.5.0 +wmill==1.536.0 \ No newline at end of file diff --git a/u/joshuakrzemien/mtg_json_download.script.yaml b/u/joshuakrzemien/mtg_json_download.script.yaml new file mode 100644 index 0000000..d2f6d40 --- /dev/null +++ b/u/joshuakrzemien/mtg_json_download.script.yaml @@ -0,0 +1,10 @@ +summary: MTG JSON Download +description: Retrieves SKU files from the MTGJSON API +lock: '!inline u/joshuakrzemien/mtg_json_download.script.lock' +concurrency_time_window_s: 0 +kind: script +schema: + $schema: 'https://json-schema.org/draft/2020-12/schema' + type: object + properties: {} + required: [] diff --git a/u/joshuakrzemien/mtgjson_flow.flow/flow.yaml b/u/joshuakrzemien/mtgjson_flow.flow/flow.yaml new file mode 100644 index 0000000..b04b876 --- /dev/null +++ b/u/joshuakrzemien/mtgjson_flow.flow/flow.yaml @@ -0,0 +1,26 @@ +summary: MTGJSON Skus ETL +description: '' +value: + modules: + - id: a + value: + type: script + input_transforms: {} + is_trigger: false + path: u/joshuakrzemien/mtg_json_download + - id: b + value: + type: script + input_transforms: + download_success: + type: javascript + expr: results.a + is_trigger: false + path: u/joshuakrzemien/mtgjson_unzip + same_worker: true +schema: + $schema: 'https://json-schema.org/draft/2020-12/schema' + type: object + order: [] + properties: {} + required: [] diff --git a/u/joshuakrzemien/mtgjson_unzip.py b/u/joshuakrzemien/mtgjson_unzip.py new file mode 100644 index 0000000..6f069ca --- /dev/null +++ b/u/joshuakrzemien/mtgjson_unzip.py @@ -0,0 +1,22 @@ +import os +import wmill +from zipfile import ZipFile + +# You can import any PyPi package. +# See here for more info: https://www.windmill.dev/docs/advanced/dependencies_in_python + +# you can use typed resources by doing a type alias to dict +#postgresql = dict + +def main(download_success): + unzip_success = {} + for zip_name, success in download_success.items(): + if success: + file_name = zip_name.replace('.zip', '') + file_path = f'./shared/{file_name}' + zip_path = f'./shared/{zip_name}' + with ZipFile(zip_path, 'r') as zip_ref: + zip_ref.extractall(file_path) + unzip_success[file_name] = True + + return unzip_success \ No newline at end of file diff --git a/u/joshuakrzemien/mtgjson_unzip.script.lock b/u/joshuakrzemien/mtgjson_unzip.script.lock new file mode 100644 index 0000000..34011fd --- /dev/null +++ b/u/joshuakrzemien/mtgjson_unzip.script.lock @@ -0,0 +1,10 @@ +# py: 3.11 +anyio==4.10.0 +certifi==2025.8.3 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.10 +sniffio==1.3.1 +typing-extensions==4.15.0 +wmill==1.536.0 \ No newline at end of file diff --git a/u/joshuakrzemien/mtgjson_unzip.script.yaml b/u/joshuakrzemien/mtgjson_unzip.script.yaml new file mode 100644 index 0000000..7a29e15 --- /dev/null +++ b/u/joshuakrzemien/mtgjson_unzip.script.yaml @@ -0,0 +1,14 @@ +summary: MTGJSON Unzip +description: '' +lock: '!inline u/joshuakrzemien/mtgjson_unzip.script.lock' +kind: script +schema: + $schema: 'https://json-schema.org/draft/2020-12/schema' + type: object + properties: + download_success: + type: object + description: '' + default: null + required: + - download_success diff --git a/u/joshuakrzemien/slick_postgresql.resource.yaml b/u/joshuakrzemien/slick_postgresql.resource.yaml new file mode 100644 index 0000000..5a633ce --- /dev/null +++ b/u/joshuakrzemien/slick_postgresql.resource.yaml @@ -0,0 +1,10 @@ +description: '' +value: + dbname: windmill_test_db + host: 192.168.1.41 + password: '$var:u/joshuakrzemien/slick_postgresql' + port: 5431 + root_certificate_pem: '' + sslmode: disable + user: postgres +resource_type: postgresql diff --git a/u/joshuakrzemien/slick_postgresql.variable.yaml b/u/joshuakrzemien/slick_postgresql.variable.yaml new file mode 100644 index 0000000..5f0cabc --- /dev/null +++ b/u/joshuakrzemien/slick_postgresql.variable.yaml @@ -0,0 +1,3 @@ +description: Token for postgresql +value: Y0FQBPsybfQXhDU7VOYO2g== +is_secret: true diff --git a/wmill-lock.yaml b/wmill-lock.yaml new file mode 100644 index 0000000..c3d8675 --- /dev/null +++ b/wmill-lock.yaml @@ -0,0 +1,5 @@ +version: v2 +locks: + u/joshuakrzemien/db_test+972c08a79a04b0017b517504a61b5d5069e82b199171e0569f281508306d8c46: 972c08a79a04b0017b517504a61b5d5069e82b199171e0569f281508306d8c46 + u/joshuakrzemien/mtg_json_download+f31539309b4d4b4ef529e47352f676f64e58e2a1b2797c2a630acab8a7c40260: f31539309b4d4b4ef529e47352f676f64e58e2a1b2797c2a630acab8a7c40260 + u/joshuakrzemien/mtgjson_unzip+0bdcf659018721653979c2de3bfd9ce2f70cf0e76ef6993226ee76e9d6c73dc4: 0bdcf659018721653979c2de3bfd9ce2f70cf0e76ef6993226ee76e9d6c73dc4