diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..1ec1939bd --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:24.04 + +# Avoid prompts from apt +ENV DEBIAN_FRONTEND=noninteractive + +# Install essential build tools and dependencies +RUN apt-get update && apt-get install -y \ + git gcc make clang libssl-dev sqlite3 libsqlite3-dev mc htop + +# Install V +RUN git clone https://github.com/vlang/v /opt/vlang \ + && cd /opt/vlang \ + && make \ + && ln -s /opt/vlang/v /usr/local/bin/v + +CMD ["redis-server"] \ No newline at end of file diff --git a/.devcontainer/check.sh b/.devcontainer/check.sh new file mode 100755 index 000000000..986a1fc75 --- /dev/null +++ b/.devcontainer/check.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# if [ -z "${DEPLOYKEY}" ]; then +# echo "ERROR: DEPLOYKEY environment variable is not set" +# exit 1 +# fi + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..f4f152a50 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,39 @@ +{ + "name": "Crystallib Development", + "build": { + "dockerfile": "Dockerfile" + }, + "workspaceFolder": "/root/code/github/freeflowuniverse/crystallib", + "features": { + "ghcr.io/devcontainers/features/common-utils:2": { + "installZsh": true, + "configureZshAsDefaultShell": true, + "installOhMyZsh": true, + "upgradePackages": true + } + }, + "customizations": { + "vscode": { + "extensions": [ + "vosca.vscode-v-analyzer", + "saoudrizwan.claude-dev", + "ms-vscode.vscode-typescript-next", + "mhutchie.git-graph" + ] + } + }, + "initializeCommand":"./.devcontainer/check.sh", + "privileged":true, + "mounts": [ + "source=${localEnv:HOME}/code,target=/root/code,type=bind,consistency=cached" + ], + "forwardPorts": [3000, 6379], + "portsAttributes": { + "3000": { + "label": "Hello Remote World", + "onAutoForward": "notify" + } + }, + "postStartCommand": "/root/code/github/freeflowuniverse/crystallib/install.sh", + "remoteUser": "root" +} diff --git a/.gitignore b/.gitignore index f920227db..c6a49afda 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,6 @@ zdb-index dump.rdb output/ *.db +.stellar vdocs/ data.ms/ - diff --git a/crystallib/clients/stellar1/factory.v b/_archive/stellar1/factory.v similarity index 100% rename from crystallib/clients/stellar1/factory.v rename to _archive/stellar1/factory.v diff --git a/crystallib/clients/stellar1/readme.md b/_archive/stellar1/readme.md similarity index 83% rename from crystallib/clients/stellar1/readme.md rename to _archive/stellar1/readme.md index 121b3a387..ea9dadf7d 100644 --- a/crystallib/clients/stellar1/readme.md +++ b/_archive/stellar1/readme.md @@ -2,7 +2,7 @@ # Stellar Client -see [examples/clients/b2_kristof.vsh](examples/clients/stellar.vsh) for example +see [examples/clients/stellar.vsh](examples/clients/stellar.vsh) for example ```v mut cl:=stellar.get(instance:"test")! diff --git a/crystallib/clients/stellar1/stellar.py b/_archive/stellar1/stellar.py similarity index 100% rename from crystallib/clients/stellar1/stellar.py rename to _archive/stellar1/stellar.py diff --git a/crystallib/clients/stellar1/stellar.v b/_archive/stellar1/stellar.v similarity index 100% rename from crystallib/clients/stellar1/stellar.v rename to _archive/stellar1/stellar.v diff --git a/cli/hero/hero.v b/cli/hero/hero.v index 1c789776a..2f6fb6230 100644 --- a/cli/hero/hero.v +++ b/cli/hero/hero.v @@ -1,8 +1,9 @@ module main import os -import cli { Command } +import cli { Command, Flag } import freeflowuniverse.crystallib.core.herocmds +import freeflowuniverse.crystallib.hero.publishing import freeflowuniverse.crystallib.installers.base as installerbase import freeflowuniverse.crystallib.installers.db.redis import freeflowuniverse.crystallib.ui.console @@ -27,13 +28,21 @@ fn do() ! { } } - mut cmd := Command{ name: 'hero' description: 'Your HERO toolset.' version: '1.0.31' } + cmd.add_flag(Flag{ + flag: .string + name: 'url' + abbrev: 'u' + global: true + description: 'url of playbook' + }) + + // herocmds.cmd_run_add_flags(mut cmd) mut toinstall:=false if !osal.cmd_exists('mc') || !osal.cmd_exists('redis-cli') { @@ -83,13 +92,15 @@ fn do() ! { herocmds.cmd_generator(mut cmd) herocmds.cmd_docsorter(mut cmd) - + cmd.add_command(publishing.cmd_publisher(pre_func)) cmd.setup() - - - cmd.parse(os.args) + cmd.parse(os.args) } fn main() { do() or { panic(err) } } + +fn pre_func(cmd Command) ! { + herocmds.plbook_run(cmd)! +} diff --git a/crystallib.code-workspace b/crystallib.code-workspace new file mode 100644 index 000000000..954a3293d --- /dev/null +++ b/crystallib.code-workspace @@ -0,0 +1,20 @@ +{ + "folders": [ + { + "path": "/root/code/github/freeflowuniverse/crystallib/crystallib" + }, + { + "path": "/root/code/github/freeflowuniverse/crystallib/aiprompts" + }, + { + "path": "/root/code/github/freeflowuniverse/crystallib/scripts" + }, + { + "path": "/root/code/github/freeflowuniverse/crystallib/examples" + }, + { + "path": "/root/code/github/freeflowuniverse/crystallib/cli" + } + ], + "settings": {} +} \ No newline at end of file diff --git a/crystallib/baobab/README.md b/crystallib/baobab/README.md deleted file mode 100644 index 2783f9def..000000000 --- a/crystallib/baobab/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# BAOBAB - -## BAse OBject and Actor Backend - diff --git a/crystallib/baobab/actor/actor.v b/crystallib/baobab/actor/actor.v deleted file mode 100644 index 9cb1722e0..000000000 --- a/crystallib/baobab/actor/actor.v +++ /dev/null @@ -1,18 +0,0 @@ -module actor - -import freeflowuniverse.crystallib.baobab.osis - -pub struct Actor { -pub mut: - osis osis.OSIS -} - -pub struct ActorConfig { - osis.OSISConfig -} - -pub fn new(config ActorConfig) !Actor { - return Actor{ - osis: osis.new(config.OSISConfig)! - } -} diff --git a/crystallib/baobab/generator/actor.v b/crystallib/baobab/generator/actor.v deleted file mode 100644 index 680a2c84a..000000000 --- a/crystallib/baobab/generator/actor.v +++ /dev/null @@ -1,294 +0,0 @@ -module generator - -import freeflowuniverse.crystallib.core.codemodel { CodeFile, CodeItem, File, Function, Import, Module, Struct } -import freeflowuniverse.crystallib.core.texttools -import freeflowuniverse.crystallib.core.codeparser -import freeflowuniverse.crystallib.data.markdownparser -import freeflowuniverse.crystallib.data.markdownparser.elements { Header } -import freeflowuniverse.crystallib.rpc.openrpc -import freeflowuniverse.crystallib.core.pathlib -import os -import json - -fn get_children(s Struct, code []CodeItem) []Struct { - structs := code.filter(it is Struct).map(it as Struct) - mut children := []Struct{} - for structure in structs { - if s.fields.any(it.typ.symbol == structure.name) { - children << structure - children << get_children(structure, code) - } - } - - return children -} - -pub fn generate_actor(name string, object_names []string, code []CodeItem) !Actor { - mut objects := []BaseObject{} - for s_ in code.filter(it is Struct).map(it as Struct).filter(texttools.name_fix(it.name) in object_names.map(texttools.name_fix(it))) { - mut s := s_ as Struct - s.mod = 'tftc.baobab.models.${s.mod}' - objects << BaseObject{ - structure: s - methods: code.filter(it is Function).map(it as Function).filter(it.receiver.typ.symbol == s.name).map(it as Function) - children: get_children(s, code) - } - } - - mut actor := Actor{ - name: name - objects: objects - // mod: generate_actor_module(name, objects)! - } - actor.structure = generate_actor_struct(name) - actor.methods = actor.generate_methods()! - return actor -} - -pub fn (actor Actor) generate_methods() ![]ActorMethod { - mut methods := []ActorMethod{} - for object in actor.objects { - methods << [ - ActorMethod{ - name: object.structure.name - func: generate_new_method(actor.structure, object) - }, - ActorMethod{ - name: object.structure.name - func: generate_get_method(actor.structure, object) - }, - ActorMethod{ - name: object.structure.name - func: generate_set_method(actor.structure, object) - }, - ActorMethod{ - name: object.structure.name - func: generate_delete_method(actor.structure, object) - }, - ActorMethod{ - name: object.structure.name - func: generate_list_method(actor.structure, object) - }, - ] - } - return methods -} - -pub struct ActorConfig { -pub mut: - name string -} - -pub fn parse_actor(path string) !Actor { - code := codeparser.parse_v(path, recursive: true)! - mut config := parse_config('${path}/config.json')! - - mut methods := []ActorMethod{} - for s in code.filter(it is Function).map(it as Function).filter(it.receiver.name == config.name) { - methods << ActorMethod{ - func: s - } - } - mut actor := Actor{} - actor.methods = methods - actor.structure = generate_actor_struct(actor.name) - - return actor -} - -pub fn parse_config(path string) !ActorConfig { - mut config_file := pathlib.get_file(path: path)! - actor := json.decode(ActorConfig, config_file.read()!)! - return actor -} - -pub fn parse_readme(path string) !Actor { - readme := markdownparser.new(path: '${path}/README.md')! - name_header := readme.children()[1] as Header - name := name_header.content - return Actor{ - name: name - } -} - -pub fn (a Actor) generate_module() !Module { - actor_struct := generate_actor_struct(a.name) - - readme := File{ - name: 'README' - extension: 'md' - content: '# ${a.name}\n${a.description}' - } - mut files := [ - generate_factory_file(a.name), - ] - // files << a.generate_model_files()! - - // generate code files for each of the objects the actor is responsible for - for object in a.objects { - files << generate_object_code(actor_struct, object) - files << generate_object_test_code(actor_struct, object)! - } - - // generate code files for each of the objects the actor is responsible for - mut methods_file := CodeFile {} - mut items := - - - return Module{ - name: a.name - files: files - misc_files: [readme] - } -} - - -pub fn generate_object_code(actor Struct, object BaseObject) CodeFile { - obj_name := texttools.name_fix_pascal_to_snake(object.structure.name) - object_type := object.structure.name - - mut items := []CodeItem{} - items = [generate_new_method(actor, object), generate_get_method(actor, object), - generate_set_method(actor, object), generate_delete_method(actor, object), - generate_list_result_struct(actor, object), generate_list_method(actor, object)] - - items << generate_object_methods(actor, object) - mut file := codemodel.new_file( - mod: texttools.name_fix(actor.name) - name: obj_name - imports: [ - Import{ - mod: object.structure.mod - types: [object_type] - }, - Import{ - mod: 'freeflowuniverse.crystallib.baobab.backend' - types: ['FilterParams'] - }, - ] - items: items - ) - - if object.structure.fields.any(it.attrs.any(it.name == 'index')) { - // can't filter without indices - filter_params := generate_filter_params(actor, object) - file.items << filter_params.map(CodeItem(it)) - file.items << generate_filter_method(actor, object) - } - - return file -} - -pub fn (a Actor) generate_openrpc_specification() !File { - openrpc_obj := a.generate_openrpc() - openrpc_json := openrpc_obj.encode()! - - openrpc_file := File{ - name: 'openrpc' - extension: 'json' - content: openrpc_json - } - - // sshrpc_files(a)! - // files << openrpc_files.map(CodeFile{...it, name: 'openrpc_${it.name}'}) - return openrpc_file -} - -pub fn (a Actor) generate_model_files() ![]CodeFile { - structs := a.objects.map(it.structure) - return a.objects.map(codemodel.new_file( - mod: texttools.name_fix(a.name) - name: '${texttools.name_fix(it.structure.name)}_model' - // imports: [Import{mod:'freeflowuniverse.crystallib.baobab.actor'}] - items: [it.structure] - )) -} - -pub fn generate_actor_module(name string, objects []BaseObject) !Module { - actor := generate_actor_struct(name) - mut files := [generate_factory_file(name)] - - // generate code files for each of the objects the actor is responsible for - for object in objects { - files << generate_object_code(actor, object) - files << generate_object_test_code(actor, object)! - } - return Module{ - name: name - files: files - } -} - -pub struct GenerateActorParams { - model_path string -} - -pub fn generate_factory_file(name string) CodeFile { - actor_struct := generate_actor_struct(name) - actor_factory := generate_actor_factory(actor_struct) - return codemodel.new_file( - mod: texttools.name_fix(name) - name: 'actor' - imports: [Import{ - mod: 'freeflowuniverse.crystallib.baobab.actor' - }] - items: [actor_struct, actor_factory] - ) -} - -pub fn generate_actor_struct(name string) Struct { - return Struct{ - is_pub: true - name: '${name.title()}' - embeds: [Struct{ - name: 'actor.Actor' - }] - } -} - -// generate_actor_factory generates the factory function for an actor -pub fn generate_actor_factory(actor Struct) Function { - mut function := codemodel.parse_function('pub fn get(config actor.ActorConfig) !${actor.name}') or { - panic(err) - } - function.body = 'return ${actor.name}{Actor: actor.new(config)!}' - return function -} - - -pub fn generate_actor_from_spec(openrpc_doc openrpc.OpenRPC) !Actor { - // Extract Actor metadata from OpenRPC info - actor_name := openrpc_doc.info.title - actor_description := openrpc_doc.info.description - - // Generate methods - mut methods := []ActorMethod{} - for method in openrpc_doc.methods { - method_code := method.to_code()! // Using provided to_code function - methods << ActorMethod{ - name: method.name - func: method_code - } - } - - // // Generate BaseObject structs from schemas - // mut objects := []BaseObject{} - // for key, schema_ref in openrpc_doc.components.schemas { - // struct_obj := schema_ref.to_code()! // Assuming schema_ref.to_code() converts schema to Struct - // // objects << BaseObject{ - // // structure: codemodel.Struct{ - // // name: struct_obj.name - // // } - // // } - // } - - // Build the Actor struct - actor := Actor{ - name: actor_name - description: actor_description - methods: methods - // objects: objects - } - - return actor -} diff --git a/crystallib/baobab/generator/actor_test.v b/crystallib/baobab/generator/actor_test.v deleted file mode 100644 index f7e83d318..000000000 --- a/crystallib/baobab/generator/actor_test.v +++ /dev/null @@ -1,17 +0,0 @@ -module generator - -import freeflowuniverse.crystallib.core.codemodel -import freeflowuniverse.crystallib.core.codeparser -import freeflowuniverse.crystallib.core.pathlib -import os - -fn test_generate_actor_struct() { - generator := ActorGenerator{ - model_name: 'TestActor' - } - - actor_struct := generator.generate_actor_struct() - assert actor_struct.name == 'TestActor' - assert actor_struct.embeds.len == 1 - assert actor_struct.embeds[0].name == 'actor.Actor' -} diff --git a/crystallib/baobab/generator/model.v b/crystallib/baobab/generator/model.v deleted file mode 100644 index 47b467c63..000000000 --- a/crystallib/baobab/generator/model.v +++ /dev/null @@ -1,31 +0,0 @@ -module generator - -import freeflowuniverse.crystallib.core.codemodel { Function, Module, Struct } -import os - -pub struct ActorGenerator { - model_name string -} - -pub struct Actor { -pub mut: - name string - description string - structure Struct - mod Module - methods []ActorMethod - objects []BaseObject -} - -pub struct ActorMethod { -pub: - name string - func Function -} - -pub struct BaseObject { -pub: - structure Struct - methods []Function - children []Struct -} diff --git a/crystallib/baobab/generator/openrpc.v b/crystallib/baobab/generator/openrpc.v deleted file mode 100644 index cfbc7c8a7..000000000 --- a/crystallib/baobab/generator/openrpc.v +++ /dev/null @@ -1,108 +0,0 @@ -module generator - -import freeflowuniverse.crystallib.core.codemodel { File, Function, Module, Struct } -import freeflowuniverse.crystallib.core.pathlib -import freeflowuniverse.crystallib.core.texttools -import freeflowuniverse.crystallib.rpc.openrpc { Components, OpenRPC } -import freeflowuniverse.crystallib.data.jsonschema { SchemaRef } - -pub fn (actor Actor) generate_openrpc_code() !Module { - openrpc_obj := actor.generate_openrpc() - openrpc_json := openrpc_obj.encode()! - - openrpc_file := File{ - name: 'openrpc' - extension: 'json' - content: openrpc_json - } - - mut methods_map := map[string]Function{} - for method in actor.methods { - methods_map[method.func.name] = method.func - } - - mut objects_map := map[string]Struct{} - for object in actor.objects { - objects_map[object.structure.name] = object.structure - } - // actor_struct := generate_actor_struct(actor.name) - actor_struct := actor.structure - - client_file := openrpc_obj.generate_client_file(objects_map)! - client_test_file := openrpc_obj.generate_client_test_file(methods_map, objects_map)! - - handler_file := openrpc_obj.generate_handler_file(actor_struct, methods_map, objects_map)! - handler_test_file := openrpc_obj.generate_handler_test_file(actor_struct, methods_map, - objects_map)! - - server_file := openrpc_obj.generate_server_file()! - server_test_file := openrpc_obj.generate_server_test_file()! - - return Module{ - files: [ - client_file, - client_test_file, - handler_file, - handler_test_file, - server_file, - server_test_file, - ] - misc_files: [openrpc_file] - } -} - -pub fn (actor Actor) generate_openrpc() OpenRPC { - mut schemas := map[string]SchemaRef{} - for obj in actor.objects { - schemas[obj.structure.name] = jsonschema.struct_to_schema(obj.structure) - for child in obj.children { - schemas[child.name] = jsonschema.struct_to_schema(child) - } - } - return OpenRPC{ - info: openrpc.Info{ - title: actor.name.title() - version: '1.0.0' - } - methods: actor.methods.map(openrpc.fn_to_method(it.func)) - components: Components{ - schemas: schemas - } - } -} - -pub fn (mut a Actor) export_playground(path string, openrpc_path string) ! { - dollar := '$' - openrpc.export_playground( - dest: pathlib.get_dir(path: '${path}/playground')! - specs: [ - pathlib.get(openrpc_path), - ] - )! - mut cli_file := pathlib.get_file(path: '${path}/command/cli.v')! - cli_file.write($tmpl('./templates/playground.v.template'))! -} - -pub fn (mut a Actor) export_command(path string) ! { - dollar := '$' - name := texttools.name_fix_pascal_to_snake(a.name) - cmd_dir := pathlib.get_dir(path: '${path}/command')! - mut cli_file := pathlib.get_file(path: '${path}/command/cli.v')! - cli_file.write($tmpl('./templates/cli.v.template'))! - - mut cmd_file := pathlib.get_file(path: '${path}/command.v')! - cmd_file.write($tmpl('./templates/cli.v.template'))! -} - -// pub fn function_to_method() - -// pub fn param_to_content_descriptor(param Param) openrpc.ContentDescriptor { -// if param.name == 'id' && param.typ.symbol == - -// return openrpc.ContentDescriptor { -// name: param.name -// summary: param.description -// required: param.is_required() -// schema: -// } -// } diff --git a/crystallib/baobab/generator/openrpc_test.v b/crystallib/baobab/generator/openrpc_test.v deleted file mode 100644 index dcca30970..000000000 --- a/crystallib/baobab/generator/openrpc_test.v +++ /dev/null @@ -1,47 +0,0 @@ -module generator - -import freeflowuniverse.crystallib.core.codemodel { Function, Param, Result, Struct, Type } -import freeflowuniverse.crystallib.rpc.openrpc - -pub fn test_generate_openrpc() ! { - actor := Actor{ - methods: [ - ActorMethod{ - func: Function{ - name: 'get_object' - params: [ - Param{ - name: 'id' - typ: Type{ - symbol: 'int' - } - }, - ] - result: Result{ - typ: Type{ - symbol: 'Object' - } - } - } - }, - ] - objects: [BaseObject{ - structure: Struct{ - name: 'Object' - } - }] - } - object := generate_openrpc(actor) - panic(object.encode()!) -} - -// pub fn param_to_content_descriptor(param Param) openrpc.ContentDescriptor { -// if param.name == 'id' && param.typ.symbol == - -// return openrpc.ContentDescriptor { -// name: param.name -// summary: param.description -// required: param.is_required() -// schema: -// } -// } diff --git a/crystallib/blockchain/stellar/horizon.v b/crystallib/blockchain/stellar/horizon.v deleted file mode 100644 index 5a447f63d..000000000 --- a/crystallib/blockchain/stellar/horizon.v +++ /dev/null @@ -1,3 +0,0 @@ -module stellar - - diff --git a/crystallib/blockchain/stellar/horizon_client.v b/crystallib/blockchain/stellar/horizon_client.v index 012263863..28e0adac7 100644 --- a/crystallib/blockchain/stellar/horizon_client.v +++ b/crystallib/blockchain/stellar/horizon_client.v @@ -2,34 +2,163 @@ module stellar import freeflowuniverse.crystallib.clients.httpconnection import json +import x.json2 pub struct HorizonClient { -pub mut: - url string +pub mut: + url string } +// Struct for the order book request +pub struct OrderBookRequest { +pub mut: + selling_asset_type string @[json: 'sellingAssetType'] + selling_asset_code string @[json: 'sellingAssetCode'] + selling_asset_issuer string @[json: 'sellingAssetIssuer'] + buying_asset_type string @[json: 'buyingAssetType'] + buying_asset_code string @[json: 'buyingAssetCode'] + buying_asset_issuer string @[json: 'buyingAssetIssuer'] + limit int +} -pub fn new_horizon_client() !HorizonClient { - mut cl:=HorizonClient{url:"https://horizon.stellar.org"} - return cl +// Struct for the order book response (you'll need to match Horizon API's response format) +pub struct OrderBook { + // Add fields based on the Horizon API order book response +pub: + bids []Order + asks []Order +} +pub struct Order { +pub: + price_r Price // Price ratio as returned by Stellar Horizon + price string + amount string +} + +// TODO: this needs to be configured to work on both networks +pub fn new_horizon_client(network StellarNetwork) !HorizonClient { + url := match network { + .mainnet { + 'https://horizon.stellar.org' + } + .testnet { + 'https://horizon-testnet.stellar.org/' + } + } + + mut cl := HorizonClient{ + url: url + } + return cl } pub fn (self HorizonClient) get_account(pubkey string) !StellarAccount { - - mut client := httpconnection.new(name: 'horizon', url: self.url)! + mut client := httpconnection.new(name: 'horizon', url: self.url)! result := client.get_json( prefix: 'accounts/${pubkey}' - debug:true - cache_disable:false + debug: true + cache_disable: false )! - mut a:=json.decode(StellarAccount, result) or { - return error('Failed to create StellarAccount: error: ${result}') - } - - //println(a) + mut a := json.decode(StellarAccount, result) or { + return error('Failed to create StellarAccount: error: ${result}') + } return a -} \ No newline at end of file +} + +pub fn (self HorizonClient) get_last_transaction(address string) !TransactionInfo { + mut client := httpconnection.new(name: 'horizon', url: self.url)! + + result := client.get_json( + prefix: 'accounts/${address}/transactions?limit=1&order=desc' + debug: true + cache_disable: false + )! + + tx := json.decode(TransactionInfo, result) or { + return error('Failed to decode TransactionInfo: error: ${result}') + } + + return tx +} + +// Function to get the order book +pub fn (self HorizonClient) get_order_book(order_book_request OrderBookRequest) !OrderBook { + // Construct the query parameters + mut query_params := map[string]json2.Any{} + mut client := httpconnection.new(name: 'horizon', url: self.url)! + + $for field in order_book_request.fields { + query_params[field.name] = order_book_request.$(field.name) + } + result := client.get_json( + prefix: 'order_book?' + url_encode(query_params) + debug: true + cache_disable: false + )! + + order_book := json.decode(OrderBook, result) or { + return error('Failed to decode OrderBook: error: ${result}') + } + + return order_book +} + +@[params] +pub struct GetOfferArgs { +pub mut: + seller string + limit int = 100 +} + +struct OffersResponse { + links RootLinks @[json: '_links'] + embedded OffersEmbedded @[json: '_embedded'] +} + +struct OffersEmbedded { + records []OfferModel +} + +pub struct OfferModel { +pub mut: + id string + paging_token string + seller string + amount string + price string + last_modified_ledger int + last_modified_time string + selling GetOfferAssetInfo + buying GetOfferAssetInfo +} + +pub struct GetOfferAssetInfo { +pub mut: + asset_type string + asset_code string + asset_issuer string +} + +// Function to list offers +pub fn (self HorizonClient) get_offers(args GetOfferArgs) ![]OfferModel { + mut client := httpconnection.new(name: 'horizon', url: self.url)! + mut query_params := map[string]json2.Any{} + $for field in args.fields { + query_params[field.name] = args.$(field.name) + } + result := client.get_json( + prefix: 'offers?' + url_encode(query_params) + debug: true + cache_disable: false + )! + + response := json.decode(OffersResponse, result) or { + return error('Failed to decode OrderBook: error: ${result}') + } + + return response.embedded.records +} diff --git a/crystallib/blockchain/stellar/horizon_model_account.v b/crystallib/blockchain/stellar/horizon_model_account.v index a1f33ce8b..03338332e 100644 --- a/crystallib/blockchain/stellar/horizon_model_account.v +++ b/crystallib/blockchain/stellar/horizon_model_account.v @@ -1,8 +1,8 @@ module stellar pub struct Links { -pub mut: - self Link +pub mut: + self Link transactions Link operations Link payments Link @@ -13,28 +13,28 @@ pub mut: } pub struct Link { -pub mut: +pub mut: href string templated bool } pub struct Thresholds { -pub mut: +pub mut: low_threshold int med_threshold int high_threshold int } pub struct Flags { -pub mut: - auth_required bool - auth_revocable bool - auth_immutable bool - auth_clawback_enabled bool +pub mut: + auth_required bool + auth_revocable bool + auth_immutable bool + auth_clawback_enabled bool } pub struct Balance { - pub mut: +pub mut: balance string limit string buying_liabilities string @@ -57,21 +57,84 @@ pub mut: @[heap] pub struct StellarAccount { pub mut: - links Links - id string - account_id string - sequence string - sequence_ledger int - sequence_time string - subentry_count int - last_modified_ledger int - last_modified_time string - thresholds Thresholds - flags Flags - balances []Balance - signers []Signer - data map[string]string - num_sponsoring int - num_sponsored int - paging_token string + links Links + id string + account_id string + sequence string + sequence_ledger int + sequence_time string + subentry_count int + last_modified_ledger int + last_modified_time string + thresholds Thresholds + flags Flags + balances []Balance + signers []Signer + data map[string]string + num_sponsoring int + num_sponsored int + paging_token string +} + +pub struct TransactionInfo { +pub: + links RootLinks @[json: '_links'] + embedded Embedded @[json: '_embedded'] +} + +pub struct RootLinks { +pub: + self Link + next Link + prev Link +} + +pub struct Embedded { +pub: + records []TransactionRecord +} + +pub struct TransactionRecord { +pub: + links RecordLinks @[json: '_links'] + id string + paging_token string + successful bool + hash string + ledger int + created_at string + source_account string + source_account_sequence string + fee_account string + fee_charged string + max_fee string + operation_count int + envelope_xdr string + result_xdr string + fee_meta_xdr string + memo_type string + signatures []string + preconditions Preconditions +} + +pub struct RecordLinks { +pub: + self Link + account Link + ledger Link + operations Link + effects Link + precedes Link + succeeds Link + transaction Link +} + +pub struct Preconditions { +pub: + timebounds PreConditionTimeBounds +} + +pub struct PreConditionTimeBounds { +pub: + min_time string } diff --git a/crystallib/blockchain/stellar/stellar_client.v b/crystallib/blockchain/stellar/stellar_client.v index 7629f1eca..008443a88 100644 --- a/crystallib/blockchain/stellar/stellar_client.v +++ b/crystallib/blockchain/stellar/stellar_client.v @@ -2,157 +2,231 @@ module stellar import os -pub struct StellarAccountKeys { -pub: - name string - public_key string - secret_key string -} +const mainnet_passphrase = 'Public Global Stellar Network ; September 2015' +const mainnet_rpc_url = 'https://soroban-rpc.mainnet.stellar.gateway.fm' -// TODO: work with enum for network +const testnet_passphrase = 'Test SDF Network ; September 2015' +const testnet_rpc_url = 'https://soroban-rpc.testnet.stellar.gateway.fm' + +pub enum StellarNetwork { + mainnet + testnet +} pub struct StellarClient { pub mut: - network string - default_assetid string // default asset contract ID, can be empty - default_from string // default account to work default_from, can be empty - default_account string // default name of the account + network StellarNetwork + account_name string + account_secret string + account_address string } @[params] -pub struct StellarClientConfig { +pub struct NewStellarClientArgs { pub: - network string - default_assetid string // contract id of the asset - default_from string - default_account string // default name of the account + network StellarNetwork = .testnet + account_name string + account_secret string @[required] + cache bool = true // If you do not want to cache account keys, set to false. If it is true and you send the same account name twice, the saved keys will be overwritten. } -pub fn new_stellar_client(config StellarClientConfig) !StellarClient { +pub fn new_client(config NewStellarClientArgs) !StellarClient { + account_address := get_address(config.account_secret)! mut cl := StellarClient{ network: config.network - default_assetid: config.default_assetid - default_from: config.default_from - default_account: config.default_account + account_name: config.account_name + account_secret: config.account_secret + account_address: account_address } - if cl.default_assetid == '' { - cl.default_assetid = cl.default_assetid_get()! + + // Cache the account keys + if config.cache { + cl.add_keys()! + } else { + remove_cached_keys(name: cl.account_name, network: cl.network)! } + return cl } @[params] -pub struct AddKeysArgs { +pub struct GetStellarClientArgs { pub: - source_account_name ?string - secret string + network StellarNetwork = .testnet + account_name string } -pub fn (mut client StellarClient) add_keys(args AddKeysArgs) ! { - mut account_name := client.default_account - - if v := args.source_account_name { - account_name = v +pub fn get_client(config GetStellarClientArgs) !StellarClient { + mut cl := StellarClient{ + network: config.network + account_name: config.account_name } - cmd := 'SOROBAN_SECRET_KEY=${args.secret} stellar keys add ${account_name} --secret-key' + mut keys := get_account_keys(cl.account_name)! + cl.account_secret = keys.secret + cl.account_address = keys.address + + return cl +} + +fn (mut client StellarClient) add_keys() ! { + cmd := 'SOROBAN_SECRET_KEY=${client.account_secret} stellar keys add ${client.account_name} --secret-key --quiet' result := os.execute(cmd) if result.exit_code != 0 { return error('Failed to add keys: ${result.output}') } } -pub fn (mut client StellarClient) account_new(name string) !StellarAccountKeys { - // Generate the keys - result := os.execute('stellar keys generate ${name} --network ${client.network}') +pub fn (mut client StellarClient) default_assetid_get() !string { + result := os.execute('stellar contract id asset --asset native --network ${client.network} --quiet') if result.exit_code != 0 { - return error('Failed to generate keys: ${result.output}') + return error('Failed to get asset contract ID: ${result.output}') } - - return client.account_keys_get(name) + return result.output.trim_space() } -pub fn (mut client StellarClient) account_keys_get(name string) !StellarAccountKeys { - // Get the public key - address_result := os.execute('stellar keys address ${name} --quiet') - if address_result.exit_code != 0 { - return error('Failed to get public key: ${address_result.output}') - } - public_key := address_result.output.trim_space() +pub struct NetworkConfig { + url string + passphrase string +} - // Get the secret key - show_result := os.execute('stellar keys show ${name} --quiet') - if show_result.exit_code != 0 { - return error('Failed to get secret key: ${show_result.output}') - } - secret_key := show_result.output.trim_space() +enum ThresholdLevel { + low + med + high +} - // Return the StellarAccountKeys struct - return StellarAccountKeys{ - name: name - public_key: public_key - secret_key: secret_key - } +pub struct Operation { + source_address string + threshold ThresholdLevel } -pub fn (mut client StellarClient) account_fund(name string) !u64 { - result := os.execute('stellar keys fund ${name} --network ${client.network}') - if result.exit_code != 0 { - return error('Failed to fund account, maybe you are not on testnet: ${result.output}') +fn (mut client StellarClient) sign_with_signers(xdr_ string, ops []Operation, signers []string) !string { + mut xdr := xdr_ + mut signers_ := signers.clone() + mut signer_address_secret := map[string]string{} // Address:Secret + mut signers_signed := map[string]bool{} // Address:Secret + signers_ << client.account_secret + + for signer in signers_ { + signer_address_secret[get_address(signer)!] = signer } - // TODO: check funding is there and return + for op in ops { + source_acc := new_horizon_client(client.network)!.get_account(op.source_address)! + mut current_weight := 0 - return 0 -} + threshold := match op.threshold { + .low { + source_acc.thresholds.low_threshold + } + .med { + source_acc.thresholds.med_threshold + } + .high { + source_acc.thresholds.high_threshold + } + } -pub fn (mut client StellarClient) default_assetid_get() !string { - result := os.execute('stellar contract id asset --asset native --network ${client.network}') - if result.exit_code != 0 { - return error('Failed to get asset contract ID: ${result.output}') + for signer in source_acc.signers { + if signers_signed[signer.key] { + current_weight += signer.weight + } + } + + for signer in source_acc.signers { + if current_weight >= threshold && current_weight > 0 { + break + } + + secret := signer_address_secret[signer.key] or { continue } + if signers_signed[signer.key] { + continue + } + + signers_signed[signer.key] = true + xdr = client.sign_tx(xdr, secret)! + current_weight += signer.weight + } } - return result.output.trim_space() + + return xdr } @[params] pub struct SendPaymentParams { - default_assetid string - default_from string - to string - amount f64 +pub mut: + asset OfferAssetType = OfferAssetType('native') + destination string @[required] + amount u64 @[required] + source_address ?string // the secret of the source account + signers []string // secret of signers } -pub fn (mut client StellarClient) payment_send(params SendPaymentParams) !string { - asset_id := if params.default_assetid == '' { - client.default_assetid - } else { - params.default_assetid - } - default_from := if params.default_from == '' { client.default_from } else { params.default_from } - cmd := 'stellar contract invoke --id ${asset_id} --source-account ${default_from} --network ${client.network} -- transfer --to ${params.to} --default_from ${default_from} --amount ${params.amount}' - result := os.execute(cmd) - if result.exit_code != 0 { - return error('Failed to send payment: ${result.output}') - } - return result.output.trim_space() -} +pub fn (mut client StellarClient) payment_send(args SendPaymentParams) !string { + // mut source_secret := client.account_secret + // if v := args.source_secret { + // source_secret = v + // } -@[params] -pub struct CheckBalanceParams { - assetid string - account_id string -} + // network_config := get_network_config(client.network)! + // TODO: add options to use different assets + mut tx := client.new_transaction_envelope(client.account_address)! + tx.add_payment_op(args)! -pub fn (mut client StellarClient) balance_check(params CheckBalanceParams) !string { - asset_id := if params.assetid == '' { client.default_assetid } else { params.assetid } - cmd := 'stellar contract invoke --id ${asset_id} --source-account ${params.account_id} --network ${client.network} -- balance --id ${params.account_id}' - result := os.execute(cmd) - if result.exit_code != 0 { - return error('Failed to check balance: ${result.output}') + mut source_address := client.account_address + if v := args.source_address { + source_address = v } - return result.output.trim_space() + + mut xdr := tx.xdr()! + xdr = client.sign_with_signers(xdr, [ + Operation{ + source_address: source_address + threshold: .med + }, + ], args.signers)! + + tx_info := client.send_tx(xdr)! + return tx_info.hash + + // cmd := 'stellar tx new payment --asset ${args.asset} --source-account ${source_secret} --destination ${args.to} --amount ${args.amount} --build-only --network ${client.network} --rpc-url ${network_config.url} --network-passphrase "${network_config.passphrase}" --quiet' + // result := os.execute(cmd) + // if result.exit_code != 0 { + // return error('Failed to send payment: ${result.output}') + // } + + // mut signers_ := args.signers.clone() + // signers_ << source_secret + // mut xdr := result.output.trim_space() + // xdr = client.sign_with_signers(xdr, [ + // Operation{ + // source_address: get_address(source_secret)! + // threshold: .med + // }, + // ], signers_)! + + // tx_info := client.send_tx(xdr)! + // return tx_info.hash } +// TODO: Check what is wrong with this method. +// @[params] +// pub struct CheckBalanceParams { +// assetid string = "native" +// account_id string +// } + +// pub fn (mut client StellarClient) balance_check(params CheckBalanceParams) !string { +// asset_id := if params.assetid == '' { client.default_assetid } else { params.assetid } +// cmd := 'stellar contract invoke --id ${asset_id} --source-account ${params.account_id} --network ${client.network} -- balance --id ${params.account_id} --quiet' +// result := os.execute(cmd) +// if result.exit_code != 0 { +// return error('Failed to check balance: ${result.output}') +// } +// return result.output.trim_space() +// } + @[params] pub struct MergeArgs { pub: @@ -161,16 +235,160 @@ pub: } pub fn (mut client StellarClient) merge_accounts(args MergeArgs) ! { - mut account_name := client.default_account + mut account_name := client.account_name if v := args.source_account_name { account_name = v } - account_keys := client.account_keys_get(account_name)! - cmd := 'stellar tx new account-merge --source-account ${account_keys.secret_key} --account ${args.address} --network ${client.network}' + account_keys := get_account_keys(account_name)! + cmd := 'stellar tx new account-merge --source-account ${account_keys.secret} --account ${args.address} --network ${client.network} --quiet' result := os.execute(cmd) if result.exit_code != 0 { return error('Failed to add keys: ${result.output}') } } + +fn (mut client StellarClient) sign_tx(tx string, signer string) !string { + network_config := get_network_config(client.network)! + + cmd := 'echo "${tx}" | stellar tx sign --sign-with-key ${signer} --network ${client.network} --rpc-url "${network_config.url}" --network-passphrase "${network_config.passphrase}" --quiet' + result := os.execute(cmd) + if result.exit_code != 0 { + return error('Failed to sign transaction: ${result.output}') + } + + return result.output.trim_space() +} + +fn (mut client StellarClient) send_tx(tx string) !TransactionRecord { + network_config := get_network_config(client.network)! + + cmd := 'echo "${tx}" | stellar tx send --network ${client.network} --rpc-url ${network_config.url} --network-passphrase "${network_config.passphrase}" --filter-logs=ERROR' + result := os.execute(cmd) + if result.exit_code != 0 { + return error('Failed to send transaction: ${result.output}') + } + + mut horizon_client := new_horizon_client(client.network)! + tx_info := horizon_client.get_last_transaction(client.account_address)! + return tx_info.embedded.records[0] +} + +@[params] +pub struct StellarCreateAccountArgs { +pub mut: + address string + starting_balance u64 + source_address ?string + signers []string +} + +pub fn (mut client StellarClient) create_account(args StellarCreateAccountArgs) !string { + mut source_address := client.account_address + if v := args.source_address { + source_address = v + } + + mut tx := client.new_transaction_envelope(client.account_address)! + tx.add_create_account_op(client.account_address, + destination: args.address + starting_balance: args.starting_balance + )! + + mut xdr := tx.xdr()! + xdr = client.sign_with_signers(xdr, [ + Operation{ + source_address: source_address + threshold: .med + }, + ], args.signers)! + + tx_info := client.send_tx(xdr)! + return tx_info.hash +} + +@[params] +pub struct AddChangeTrustArgs { +pub mut: + asset_code string @[required] + issuer string @[required] + limit u64 = (u64(1) << 63) - 1 + source_address ?string + signers []string +} + +pub fn (mut client StellarClient) add_trust_line(args AddChangeTrustArgs) !string { + mut tx := client.new_transaction_envelope(client.account_address)! + tx.add_change_trust_op(args)! + + mut source_address := client.account_address + if v := args.source_address { + source_address = v + } + + mut xdr := tx.xdr()! + xdr = client.sign_with_signers(xdr, [ + Operation{ + source_address: source_address + threshold: .med + }, + ], args.signers)! + + tx_info := client.send_tx(xdr)! + return tx_info.hash +} + +@[required] +pub struct OfferArgs { +pub mut: + sell bool + buy bool + source_address ?string + selling OfferAssetType + buying OfferAssetType + amount f64 @[required] // in stroops + price f32 @[required] // Price of 1 unit of selling in terms of buying + signers []string +} + +fn (mut client StellarClient) make_offer(offer_id u64, args OfferArgs) !TransactionRecord { + if args.sell == args.buy { + return error('You must either sell or buy at the same time') + } + + mut source_address := client.account_address + if v := args.source_address { + source_address = v + } + + mut tx := client.new_transaction_envelope(client.account_address)! + tx.make_offer_op(offer_id: offer_id, offer: args, sell: args.sell, buy: args.buy)! + mut xdr := tx.xdr()! + xdr = client.sign_with_signers(xdr, [ + Operation{ + source_address: source_address + threshold: .med + }, + ], args.signers)! + return client.send_tx(xdr)! +} + +pub fn (mut client StellarClient) create_offer(args OfferArgs) !MakeOfferResult { + tx_record := client.make_offer(0, args)! + return get_offer_id_from_result_xdr(tx_record.result_xdr)! +} + +pub fn (mut client StellarClient) update_offer(offer_id u64, args OfferArgs) ! { + if args.amount == 0 { + return error('Amount must be greater than 0') + } + + client.make_offer(offer_id, args)! +} + +pub fn (mut client StellarClient) delete_offer(offer_id u64, args_ OfferArgs) ! { + mut args := args_ + args.amount = 0 + client.make_offer(offer_id, args)! +} diff --git a/crystallib/blockchain/stellar/stellar_signing.v b/crystallib/blockchain/stellar/stellar_signing.v index f3cee02a3..e6d444015 100644 --- a/crystallib/blockchain/stellar/stellar_signing.v +++ b/crystallib/blockchain/stellar/stellar_signing.v @@ -1,112 +1,121 @@ module stellar -import os - -@[params] -pub struct SignersAddArgs { -pub mut: - name string // name to get source account from - pubkeys []string -} - @[params] -pub struct AddSignerArgs { +pub struct AddSignersArgs { pub: - source_account_name ?string - address string - weight int = 1 + source_address ?string + signers_to_add []TXSigner + signers []string } -pub fn (mut client StellarClient) add_signer(args AddSignerArgs) ! { - if args.weight == 0 { - return error('a signer weight of 0 will remove signer. use remove_signer method to remove signer') +pub fn (mut client StellarClient) add_signers(args AddSignersArgs) !string { + mut ops := []Operation{} + mut source_address := client.account_address + if v := args.source_address { + source_address = v } - mut account_name := client.default_account + mut tx := client.new_transaction_envelope(client.account_address)! + for signer in args.signers_to_add { + ops << Operation{ + source_address: source_address + threshold: .high + } - if v := args.source_account_name { - account_name = v - } + if signer.key == tx.tx.source_account { + tx.add_set_options_op( + source_account: if v := args.source_address { v } else { none } + set_options: SetOptions{ + master_weight: signer.weight + } + )! + continue + } - account_keys := client.account_keys_get(account_name)! - cmd := 'stellar tx new set-options --source-account ${account_keys.secret_key} --signer ${args.address} --signer-weight ${args.weight} --network ${client.network}' - result := os.execute(cmd) - if result.exit_code != 0 { - return error('transaction failed: ${result.output}') + tx.add_signer( + signer: signer + )! } + mut xdr := tx.xdr()! + xdr = client.sign_with_signers(xdr, ops, args.signers)! + tx_info := client.send_tx(xdr)! + return tx_info.hash } @[params] -pub struct RemoveSignerArgs { -pub: - source_account_name ?string - address string +pub struct UpdateThresholdArgs { +pub mut: + source_address ?string + low_threshold ?int + med_threshold ?int + high_threshold ?int + signers []string } -pub fn (mut client StellarClient) remove_signer(args RemoveSignerArgs) ! { - mut account_name := client.default_account - - if v := args.source_account_name { - account_name = v +pub fn (mut client StellarClient) update_threshold(args UpdateThresholdArgs) !string { + if args.low_threshold == none && args.med_threshold == none && args.high_threshold == none { + return error('at least one threshold must be set') } - account_keys := client.account_keys_get(account_name)! - cmd := 'stellar tx new set-options --source-account ${account_keys.secret_key} --signer ${args.address} --signer-weight 0 --network ${client.network}' - result := os.execute(cmd) - if result.exit_code != 0 { - return error('transaction failed: ${result.output}') + mut source_address := client.account_address + if v := args.source_address { + source_address = v } -} -pub fn (mut client StellarClient) signers_add(args SignersAddArgs) ! { - jsondata := ' - { - "source_account": "GDQP2KPQGKIHYJGXNUIYOMHARUARCA7DRQSV5HRTJNBGKZ2G24AN4IQS", - "operations": [ - { - "type": "setOptions", - "signer": { - "ed25519PublicKey": "GCFDZK7XWETBVW7LYP2NNCFTELIG44QJJN6O7Q7VRP6WUIXTD6NVZ2GS", - "weight": 1 - } - }, - { - "type": "setOptions", - "signer": { - "ed25519PublicKey": "GBAQ3GILDSCFDJFGWSHDTXPOJL4FGSKGK3HDJZPZWIR7LU3JMPCRH2W7", - "weight": 1 - } - } - ], - "fee": 200, - "sequence_number": "123456789", - "memo": { - "type": "none" - }, - "time_bounds": { - "min_time": 0, - "max_time": 0 + mut tx := client.new_transaction_envelope(client.account_address)! + tx.add_set_options_op( + source_account: if v := args.source_address { v } else { none } + set_options: SetOptions{ + low_threshold: args.low_threshold + med_threshold: args.med_threshold + high_threshold: args.high_threshold } - } - ' - xdrpath := '/tmp/add-multiple-signers.xdr' - os.write_file(xdrpath, jsondata)! - result := os.execute('stellar xdr from-json --input add-multiple-signers.json --output ${xdrpath} --network ${client.network}') - if result.exit_code != 0 { - return error('Failed to convert JSON to XDR: ${result.output}') - } - result2 := os.execute('stellar tx sign --input ${xdrpath} --secret SECRET_KEY --output signed-${xdrpath} --network ${client.network}') - if result2.exit_code != 0 { - return error('Failed to sign transaction: ${result2.output}') - } + )! + + mut xdr := tx.xdr()! + xdr = client.sign_with_signers(xdr, [ + Operation{ + source_address: source_address + threshold: .high + }, + ], args.signers)! + + tx_info := client.send_tx(xdr)! + return tx_info.hash +} - result3 := os.execute('stellar tx submit --input signed-${xdrpath} --network ${client.network}') - if result3.exit_code != 0 { - return error('Failed to submit transaction: ${result3.output}') +@[params] +pub struct RemoveSignerArgs { +pub: + source_address ?string + address string + signers []string +} + +pub fn (mut client StellarClient) remove_signer(args RemoveSignerArgs) !string { + mut source_address := client.account_address + if v := args.source_address { + source_address = v } - // TODO: now check the status of the account to see if the signing has been added + mut tx := client.new_transaction_envelope(client.account_address)! + tx.add_set_options_op( + source_account: if v := args.source_address { v } else { none } + set_options: SetOptions{ + signer: TXSigner{ + key: args.address + weight: 0 + } + } + )! - os.rm(xdrpath)! - os.rm('signed-${xdrpath}')! + mut xdr := tx.xdr()! + xdr = client.sign_with_signers(xdr, [ + Operation{ + source_address: source_address + threshold: .high + }, + ], args.signers)! + tx_info := client.send_tx(xdr)! + return tx_info.hash } diff --git a/crystallib/blockchain/stellar/tradingbot/buy.v b/crystallib/blockchain/stellar/tradingbot/buy.v new file mode 100644 index 000000000..be5ccf777 --- /dev/null +++ b/crystallib/blockchain/stellar/tradingbot/buy.v @@ -0,0 +1,84 @@ +module tradingbot + +import freeflowuniverse.crystallib.blockchain.stellar +import math + +@[params] +struct BuyLowArgs { + order_book stellar.OrderBook @[required] + active_offer ?stellar.OfferModel +} + +// Buy if price is below target +fn (mut bot StellarTradingBot) buy_low(args BuyLowArgs) ! { + log('Creating/Updating a new buy offer', false) + + mut asset_info := stellar.GetOfferAssetInfo{ + asset_type: bot.buying_asset_type + asset_code: bot.buying_asset_code + asset_issuer: bot.buying_asset_issuer + } + + asset_balance := bot.get_asset_balance(asset_info)! + log('Asset ${bot.buying_asset_code} balance: ${asset_balance}', false) + + if asset_balance <= bot.preserve { + return error('Wallet does not have enough balance for asset ${bot.buying_asset_code} to make a new buy offer, current balance is ${asset_balance}.') + } + + mut buying_price := bot.buying_target_price + lowest_price := stellar.fetch_lowest_ask_price(args.order_book) // 500 + + lowest_price_float := f32(lowest_price.n) / f32(lowest_price.d) + + if lowest_price_float < bot.buying_target_price { + buying_price = lowest_price_float + } + + mut spendable_balance := asset_balance - bot.preserve + log('Spendable balance: ${spendable_balance}', false) + + mut amount := math.min(spendable_balance, bot.buying_amount) + + offer_args := stellar.OfferArgs{ + selling: stellar.get_offer_asset_type(bot.buying_asset_type, bot.buying_asset_code, + bot.buying_asset_issuer) + buying: stellar.get_offer_asset_type(bot.selling_asset_type, bot.selling_asset_code, + bot.selling_asset_issuer) + amount: amount + buy: true + price: f32(buying_price) + } + + if active_offer := args.active_offer { + // check if update is needed + amount = round_to_precision(amount, 7) + active_offer_amount := f64(round_to_precision(active_offer.amount.f64(), 7)) + active_offer_price := round_to_precision(active_offer.price.f64(), 7) + buying_price = f64(round_to_precision(f64(buying_price), 7)) + + log('active offer: price: ${active_offer_price} - amount: ${active_offer_amount}', + false) + log('buying: price: ${buying_price} - amount: ${amount}', false) + + if active_offer_price == buying_price && math.abs(active_offer_amount - amount) < 1e-7 { + // don't need an update + log('offer ${active_offer.id.int()} is up-to-date.', false) + return + } + + bot.sclient.update_offer(active_offer.id.u64(), offer_args)! + log('Offer ${active_offer.id.int()} is updated', false) + + return + } else { + mut offer_result := bot.sclient.create_offer(offer_args)! + if offer_result.claimed { + log('Offer created and claimed by ${offer_result.offer_id}', false) + } else { + log('Offer ${offer_result.offer_id} is created', false) + } + + return + } +} diff --git a/crystallib/blockchain/stellar/tradingbot/delete.v b/crystallib/blockchain/stellar/tradingbot/delete.v new file mode 100644 index 000000000..5663d2943 --- /dev/null +++ b/crystallib/blockchain/stellar/tradingbot/delete.v @@ -0,0 +1,38 @@ +module tradingbot + +import freeflowuniverse.crystallib.blockchain.stellar +import freeflowuniverse.crystallib.ui.console + +fn (mut bot StellarTradingBot) delete_sell_offers(mut active_offers []stellar.OfferModel) ! { + for mut offer in active_offers { + bot.delete_offer(offer: offer, sell: true)! + } +} + +fn (mut bot StellarTradingBot) delete_buy_offers(mut active_offers []stellar.OfferModel) ! { + for mut offer in active_offers { + bot.delete_offer(offer: offer, buy: true)! + } +} + +@[params] +struct DeleteOfferArgs { +pub mut: + offer stellar.OfferModel @[required] + sell bool + buy bool +} + +fn (mut bot StellarTradingBot) delete_offer(args DeleteOfferArgs) ! { + console.print_stderr('Deleting offer ${args.offer.id.u64()}') + bot.sclient.delete_offer(args.offer.id.u64(), stellar.OfferArgs{ + amount: 0 + price: args.offer.price.f32() + selling: stellar.get_offer_asset_type(bot.selling_asset_type, bot.selling_asset_code, + bot.selling_asset_issuer) + buying: stellar.get_offer_asset_type(bot.buying_asset_type, bot.buying_asset_code, + bot.buying_asset_issuer) + buy: args.buy + sell: args.sell + })! +} diff --git a/crystallib/blockchain/stellar/tradingbot/models.v b/crystallib/blockchain/stellar/tradingbot/models.v new file mode 100644 index 000000000..8bfbf51c4 --- /dev/null +++ b/crystallib/blockchain/stellar/tradingbot/models.v @@ -0,0 +1,32 @@ +module tradingbot + +import freeflowuniverse.crystallib.blockchain.stellar + +pub enum StellarTradingBotOperation { + sell + buy +} + +pub struct StellarTradingBot { +mut: + hclient stellar.HorizonClient // Horizon client + sclient stellar.StellarClient // Stellar client +pub mut: + account_secret string // private key + account_address string // public key + + selling_asset_code string // asset to sell + selling_asset_issuer string // issuer of the asset to sell + selling_asset_type string // type of the asset to sell + buying_asset_code string // asset to buy + buying_asset_issuer string // issuer of the asset to buy + buying_asset_type string // type of the asset to buy + // selling stellar.OfferAssetType + // buying stellar.OfferAssetType + + buying_target_price f64 // price to buy at + selling_target_price f64 // price to sell at + selling_amount f64 + buying_amount f64 + preserve f64 // min balance to have in account +} diff --git a/crystallib/blockchain/stellar/tradingbot/sell.v b/crystallib/blockchain/stellar/tradingbot/sell.v new file mode 100644 index 000000000..989c35d68 --- /dev/null +++ b/crystallib/blockchain/stellar/tradingbot/sell.v @@ -0,0 +1,81 @@ +module tradingbot + +import freeflowuniverse.crystallib.blockchain.stellar +import math + +@[params] +struct SellHighArgs { + order_book stellar.OrderBook @[required] + active_offer ?stellar.OfferModel +} + +// Sell if price is above target +fn (mut bot StellarTradingBot) sell_high(args SellHighArgs) ! { + mut asset_info := stellar.GetOfferAssetInfo{ + asset_type: bot.selling_asset_type + asset_code: bot.selling_asset_code + asset_issuer: bot.selling_asset_issuer + } + + mut asset_balance := bot.get_asset_balance(asset_info)! + log('Asset ${bot.selling_asset_code} balance: ${asset_balance}', true) + + if asset_balance <= bot.preserve { + return error('Wallet does not have enough balance for asset ${bot.selling_asset_code} to make a new sell offer, current balance is ${asset_balance}.') + } + + mut selling_price := bot.selling_target_price + + highest_price := stellar.fetch_highest_bid_price(args.order_book) + highest_price_float := f32(highest_price.n) / f32(highest_price.d) + + if highest_price_float > bot.selling_target_price { + selling_price = highest_price_float + } + + spendable_balance := asset_balance - bot.preserve + log('Spendable balance: ${spendable_balance}', true) + + mut amount := math.min(spendable_balance, bot.selling_amount) + + offer_args := stellar.OfferArgs{ + selling: stellar.get_offer_asset_type(bot.selling_asset_type, bot.selling_asset_code, + bot.selling_asset_issuer) + buying: stellar.get_offer_asset_type(bot.buying_asset_type, bot.buying_asset_code, + bot.buying_asset_issuer) + amount: amount + sell: true + price: f32(selling_price) + } + + if active_offer := args.active_offer { + // check if update is needed + amount = round_to_precision(amount, 7) + active_offer_amount := f64(round_to_precision(active_offer.amount.f64(), 7)) + active_offer_price := round_to_precision(active_offer.price.f64(), 7) + selling_price = f64(round_to_precision(f64(selling_price), 7)) + + log('active offer: price: ${active_offer_price} - amount: ${active_offer_amount}', + true) + log('selling price: ${selling_price} - amount: ${amount}', true) + + if active_offer_price == selling_price && math.abs(active_offer_amount - amount) < 1e-7 { + // don't need an update + log('offer ${active_offer.id.int()} is up-to-date.', true) + return + } + + bot.sclient.update_offer(active_offer.id.u64(), offer_args)! + log('Offer ${active_offer.id.int()} is updated', true) + + return + } else { + mut offer_result := bot.sclient.create_offer(offer_args)! + if offer_result.claimed { + log('Offer created and claimed by ${offer_result.offer_id}', true) + } else { + log('Offer ${offer_result.offer_id} is created', true) + } + return + } +} diff --git a/crystallib/blockchain/stellar/tradingbot/trade.v b/crystallib/blockchain/stellar/tradingbot/trade.v new file mode 100644 index 000000000..c129d8cd7 --- /dev/null +++ b/crystallib/blockchain/stellar/tradingbot/trade.v @@ -0,0 +1,252 @@ +module tradingbot + +import freeflowuniverse.crystallib.blockchain.stellar +import freeflowuniverse.crystallib.ui.console +import time + +const poll_interval = 10 * time.second // Polling interval for bot operations + +@[params] +pub struct StellarTradingBotArgs { +pub mut: + account_secret string @[required] // The account secret + + selling_asset_code string // asset to sell + selling_asset_issuer string + selling_asset_type string + buying_asset_code string // asset to buy + buying_asset_issuer string + buying_asset_type string + + selling_target_price f64 @[required] // Your desired sell price + selling_amount f64 + buying_target_price f64 @[required] // Your desired buy price + buying_amount f64 + network stellar.StellarNetwork = .testnet +} + +// Initialize the bot +pub fn new(args_ StellarTradingBotArgs) !StellarTradingBot { + console.print_header('Initializing trading bot...') + mut args := args_ + if args.selling_asset_type == '' { + args.selling_asset_type = determine_asset_type(args.selling_asset_code) + } + + if args.buying_asset_type == '' { + args.buying_asset_type = determine_asset_type(args.buying_asset_type) + } + + if args.selling_target_price <= args.buying_target_price { + return error('selling price must be strictly higher than buying price') + } + + mut hclient := stellar.new_horizon_client(args.network)! + mut sclient := stellar.new_client( + account_name: 'tradingbot' + account_secret: args.account_secret + network: args.network + cache: false + )! + + account_keys := stellar.get_account_keys(args.account_secret)! + + mut bot := StellarTradingBot{ + hclient: hclient + sclient: sclient + account_secret: account_keys.secret + account_address: account_keys.address + selling_asset_type: args.selling_asset_type + selling_asset_code: args.selling_asset_code + selling_asset_issuer: args.selling_asset_issuer + selling_amount: args.selling_amount + selling_target_price: args.selling_target_price + buying_target_price: args.buying_target_price + buying_asset_code: args.buying_asset_code + buying_asset_type: args.buying_asset_type + buying_asset_issuer: args.buying_asset_issuer + buying_amount: args.buying_amount + } + + // bot.update_assets() + bot.add_needed_trust_lines()! + return bot +} + +// Add trust lines for the assets +fn (mut bot StellarTradingBot) add_needed_trust_lines() ! { + account := bot.hclient.get_account(bot.account_address)! + + mut need_selling_trustline, mut need_buying_trustline := bot.selling_asset_type != 'native', bot.buying_asset_type != 'native' + for balance in account.balances { + if balance.asset_type == bot.selling_asset_type + && balance.asset_code == bot.selling_asset_code + && balance.asset_issuer == bot.selling_asset_issuer { + need_selling_trustline = false + } + + if balance.asset_type == bot.buying_asset_type + && balance.asset_code == bot.buying_asset_code + && balance.asset_issuer == bot.buying_asset_issuer { + need_buying_trustline = false + } + } + + if need_selling_trustline { + console.print_header('Adding trustline for ${bot.selling_asset_code}, Issuer: ${bot.selling_asset_issuer}') + bot.sclient.add_trust_line( + asset_code: bot.selling_asset_code + issuer: bot.selling_asset_issuer + )! + } + + if need_buying_trustline { + console.print_header('Adding trustline for ${bot.buying_asset_code}, Issuer: ${bot.buying_asset_issuer}') + bot.sclient.add_trust_line( + asset_code: bot.buying_asset_code + issuer: bot.buying_asset_issuer + )! + } +} + +// Runs a specific operation (buy or sell) in a loop +pub fn (mut bot StellarTradingBot) run() ! { + mut selling_asset := bot.selling_asset_code + mut buying_asset := bot.buying_asset_code + + if bot.selling_asset_type == 'native' { + selling_asset = 'XLM' + } + + if bot.buying_asset_type == 'native' { + buying_asset = 'XLM' + } + + console.print_header('Bot status: selling ${selling_asset}, buying ${buying_asset}') + + mut active_offers := bot.fetch_wallet_offers()! + + if bot.selling_amount == 0 { + bot.delete_sell_offers(mut active_offers)! + } + + if bot.buying_amount == 0 { + bot.delete_buy_offers(mut active_offers)! + } + + if bot.selling_amount == 0 && bot.buying_amount == 0 { + return + } + + for { + active_offers = bot.fetch_wallet_offers()! + + if active_offers.len > 1 { + return error('Wallet has more than one offer') + } + + order_book := bot.fetch_order_book() or { + console.print_stderr('failed to get orderbook: ${err}') + continue + } + + active_offer := fn (active_offers []stellar.OfferModel) ?stellar.OfferModel { + if active_offers.len == 1 { + return active_offers[0] + } else { + return none + } + }(active_offers) + + if bot.should_sell(order_book) { + console.print_header('Highest bid price is more than buying threshold. Trying to sell ${selling_asset} and buy ${buying_asset}...') + bot.sell_high(active_offer: active_offer, order_book: order_book) or { + console.print_stderr('${err}') + } + } else { + console.print_header('Lowest ask price is less than or equal to buying threshold. Trying to buy ${buying_asset} and sell ${selling_asset}') + bot.buy_low(active_offer: active_offer, order_book: order_book) or { + console.print_stderr('${err}') + } + } + + // Adjust polling interval as needed + time.sleep(tradingbot.poll_interval) + } +} + +fn (mut bot StellarTradingBot) should_sell(order_book stellar.OrderBook) bool { + highest_price := stellar.fetch_highest_bid_price(order_book) + highest_price_float := f32(highest_price.n) / f32(highest_price.d) + + return highest_price_float > bot.buying_target_price +} + +// Fetch order book +fn (mut bot StellarTradingBot) fetch_order_book() !stellar.OrderBook { + mut order_book_request := stellar.OrderBookRequest{ + selling_asset_code: bot.selling_asset_code + selling_asset_type: bot.selling_asset_type + buying_asset_code: bot.buying_asset_code + buying_asset_type: bot.buying_asset_type + selling_asset_issuer: bot.selling_asset_issuer + buying_asset_issuer: bot.buying_asset_issuer + limit: 200 + } + + order_book := bot.hclient.get_order_book(order_book_request) or { + return error('Failed to fetch order book: ${err}') + } + return order_book +} + +fn (mut bot StellarTradingBot) fetch_wallet_offers() ![]stellar.OfferModel { + // Fetch offers from Horizon client + mut offers_page := bot.hclient.get_offers(seller: bot.account_address, limit: 200)! + + // Filter offers to find the matching pair + mut matching_offers := []stellar.OfferModel{} + + for mut offer in offers_page { + if offer.selling.asset_code == bot.selling_asset_code + && offer.selling.asset_issuer == bot.selling_asset_issuer + && offer.selling.asset_type == bot.selling_asset_type + && offer.buying.asset_code == bot.buying_asset_code + && offer.buying.asset_issuer == bot.buying_asset_issuer + && offer.buying.asset_type == bot.buying_asset_type { + matching_offers << offer + } + } + + return matching_offers +} + +fn (mut bot StellarTradingBot) get_asset_balance(asset stellar.GetOfferAssetInfo) !f64 { + account := bot.hclient.get_account(bot.account_address)! + + for balance_info in account.balances { + if asset.asset_type == 'native' && balance_info.asset_type == 'native' { + return balance_info.balance.f64() + } + + if balance_info.asset_code == asset.asset_code + && balance_info.asset_issuer == asset.asset_issuer + && asset.asset_type == balance_info.asset_type { + return balance_info.balance.f64() + } + } + + return error('account does not have trust line for asset ${asset.asset_code}') +} + +fn (mut bot StellarTradingBot) match_sell_asset(asset_type string, asset_code string, asset_issuer string) bool { + return (bot.selling_asset_type == 'native' && asset_type == 'native') + || (bot.selling_asset_type == asset_type && bot.selling_asset_code == asset_code + && bot.selling_asset_issuer == asset_issuer) +} + +fn (mut bot StellarTradingBot) match_buy_asset(asset_type string, asset_code string, asset_issuer string) bool { + return (bot.buying_asset_type == 'native' && asset_type == 'native') + || (bot.buying_asset_type == asset_type && bot.buying_asset_code == asset_code + && bot.buying_asset_issuer == asset_issuer) +} diff --git a/crystallib/blockchain/stellar/tradingbot/utils.v b/crystallib/blockchain/stellar/tradingbot/utils.v new file mode 100644 index 000000000..f113e702d --- /dev/null +++ b/crystallib/blockchain/stellar/tradingbot/utils.v @@ -0,0 +1,27 @@ +module tradingbot + +import math +import freeflowuniverse.crystallib.ui.console + +// Determines the Stellar asset type based on code length +fn determine_asset_type(asset_code string) string { + return if asset_code.len <= 4 { + 'credit_alphanum4' + } else { + 'credit_alphanum12' + } +} + +// Rounding function, to the specified precision +fn round_to_precision(num f64, precision int) f64 { + factor := math.pow(10, precision) + return math.round(num * factor) / factor +} + +fn log(message string, sell bool) { + if sell { + console.cprintln(foreground: .light_green, text: '|Sell logs| ${message}') + } else { + console.cprintln(foreground: .light_blue, text: '|Buy logs| ${message}') + } +} diff --git a/crystallib/blockchain/stellar/transaction.v b/crystallib/blockchain/stellar/transaction.v new file mode 100644 index 000000000..2743c4fbf --- /dev/null +++ b/crystallib/blockchain/stellar/transaction.v @@ -0,0 +1,322 @@ +module stellar + +import x.json2 + +pub struct TimeBounds { +pub: + min_time u64 + max_time u64 +} + +pub struct Condition { +pub: + time TimeBounds +} + +@[params] +pub struct TXSigner { +pub: + key string + weight int = 1 +} + +pub struct SetOptions { +pub mut: + inflation_dest ?string + clear_flags ?int + set_flags ?int + master_weight ?int + low_threshold ?int + med_threshold ?int + high_threshold ?int + home_domain ?string + signer ?TXSigner +} + +// a placeholder. +pub struct PaymentOptions { +pub mut: + destination string + amount u64 + asset OfferAssetType +} + +@[noinit] +pub struct OperationBody { +pub mut: + set_options ?SetOptions + create_account ?TXCreateAccount + payment ?PaymentOptions + change_trust ?ChangeTrust + manage_sell_offer ?Offer + manage_buy_offer ?Offer +} + +pub struct TransactionOperation { +pub mut: + source_account ?string + body OperationBody +} + +pub struct Transaction { +pub mut: + source_account string + fee int + seq_num u64 + cond Condition + memo string = 'none' + operations []TransactionOperation + ext string = 'v0' +} + +pub struct ChangeTrust { +pub mut: + line AssetType + limit ?u64 +} + +pub struct Price { +pub mut: + n int + d int +} + +pub type OfferAssetType = AssetType | string + +pub struct Offer { +pub mut: + selling OfferAssetType + buying OfferAssetType + amount ?u64 // stroops + buy_amount ?u64 // stroops + price Price + offer_id u64 +} + +fn (mut tx TransactionEnvelope) add_change_trust_op(args AddChangeTrustArgs) ! { + if args.asset_code.len > 12 { + return error('asset code must be less than 12 bytes') + } + + asset := Asset{ + asset_code: args.asset_code + issuer: args.issuer + } + + mut change_trust_line := AssetType{} + if args.asset_code.len <= 4 { + change_trust_line.credit_alphanum4 = asset + } else { + change_trust_line.credit_alphanum12 = asset + } + + body := OperationBody{ + change_trust: ChangeTrust{ + line: change_trust_line + limit: args.limit + } + } + + tx.add_operation(args.source_address, body)! + tx.tx.fee += 100 +} + +fn (mut tx TransactionEnvelope) add_payment_op(args SendPaymentParams) ! { + body := OperationBody{ + payment: PaymentOptions{ + destination: args.destination + asset: args.asset + amount: args.amount + } + } + + tx.add_operation(args.source_address, body)! + tx.tx.fee += 100 +} + +pub struct AssetType { +pub mut: + credit_alphanum4 ?Asset + credit_alphanum12 ?Asset +} + +pub fn new_asset_type(code string, issuer string) AssetType { + asset := Asset{ + asset_code: code + issuer: issuer + } + + if code.len <= 4 { + return AssetType{ + credit_alphanum4: asset + } + } + + return AssetType{ + credit_alphanum12: asset + } +} + +pub struct Asset { +pub mut: + asset_code string + issuer string +} + +fn (mut c StellarClient) new_transaction_envelope(source_account_address string) !TransactionEnvelope { + hcl := new_horizon_client(c.network)! + account := hcl.get_account(source_account_address)! + + sequence_number := account.sequence.u64() + 1 + + return TransactionEnvelope{ + tx: Transaction{ + source_account: source_account_address + seq_num: sequence_number + } + } +} + +pub struct TransactionEnvelope { +pub mut: + tx Transaction + signatures []string +} + +// struct TransactionArgs { +// operation +// } + +fn (mut tx TransactionEnvelope) add_operation(source_account ?string, op OperationBody) ! { + mut ops := 0 + + $for field in op.fields { + if op.$(field.name) != none { + ops += 1 + } + } + if ops != 1 { + return error('only one operation type must be added per operation, found ${ops}') + } + + tx.tx.operations << TransactionOperation{ + source_account: source_account + body: op + } +} + +@[params] +pub struct TXAddSignerArgs { +pub: + source_account ?string + signer TXSigner +} + +fn (mut tx TransactionEnvelope) add_signer(args TXAddSignerArgs) ! { + body := OperationBody{ + set_options: SetOptions{ + signer: args.signer + } + } + + tx.add_operation(args.source_account, body)! + tx.tx.fee += 100 +} + +@[params] +pub struct TXAddSetOptionsOperationArgs { + source_account ?string + set_options SetOptions +} + +fn (mut tx TransactionEnvelope) add_set_options_op(args TXAddSetOptionsOperationArgs) ! { + body := OperationBody{ + set_options: args.set_options + } + + tx.add_operation(args.source_account, body)! + tx.tx.fee += 100 +} + +fn (tx TransactionEnvelope) xdr() !string { + json_encoding := json2.encode({ + 'tx': tx + }) + + return encode_tx_to_xdr(json_encoding)! +} + +// Struct for the "create_account" request +@[params] +pub struct TXCreateAccount { +pub mut: + destination string @[required] // The public key of the account to create + starting_balance u64 @[required] // Use f64 for the raw balance (in this case, 100.0) +} + +fn (mut tx TransactionEnvelope) add_create_account_op(source_account ?string, args TXCreateAccount) ! { + body := OperationBody{ + create_account: args + } + + tx.add_operation(source_account, body)! + tx.tx.fee += 100 +} + +pub fn get_offer_asset_type(asset_type string, asset_code string, asset_issuer string) OfferAssetType { + if asset_type == 'native' { + return OfferAssetType('native') + } + + mut asset := AssetType{} + if asset_code.len <= 4 { + asset.credit_alphanum4 = Asset{ + asset_code: asset_code + issuer: asset_issuer + } + } else { + asset.credit_alphanum12 = Asset{ + asset_code: asset_code + issuer: asset_issuer + } + } + + return OfferAssetType(asset) +} + +@[params] +pub struct MakeOfferOpArgs { + offer_id u64 + offer OfferArgs + sell bool + buy bool +} + +fn (mut tx TransactionEnvelope) make_offer_op(args MakeOfferOpArgs) ! { + if args.sell == args.buy { + return error('You must either sell or buy at the same time') + } + + // selling_asset_type := get_offer_asset_type(args.offer.selling) + // buying_asset_type := get_offer_asset_type(args.offer.buying) + + mut offer := Offer{ + selling: args.offer.selling + buying: args.offer.buying + price: get_offer_price(args.offer.price) + offer_id: args.offer_id + } + + mut body := OperationBody{} + + if args.sell { + offer.amount = u64(args.offer.amount * 1e7) + body.manage_sell_offer = offer + } else { + offer.buy_amount = u64(args.offer.amount * 1e7) + body.manage_buy_offer = offer + } + + tx.add_operation(args.offer.source_address, body)! + tx.tx.fee += 100 +} diff --git a/crystallib/blockchain/stellar/utils.v b/crystallib/blockchain/stellar/utils.v new file mode 100644 index 000000000..8af624778 --- /dev/null +++ b/crystallib/blockchain/stellar/utils.v @@ -0,0 +1,285 @@ +module stellar + +import freeflowuniverse.crystallib.clients.httpconnection +import os +import math +import x.json2 + +pub struct StellarAccountKeys { +pub: + name string + address string + secret string +} + +pub fn get_address(secret string) !string { + cmd := 'stellar keys address ${secret} --quiet' + result := os.execute(cmd) + if result.exit_code != 0 { + return error('Failed to get address: ${result.output}') + } + + return result.output.trim_space() +} + +pub fn get_account_keys(name string) !StellarAccountKeys { + // Get the public key + address_result := os.execute('stellar keys address ${name} --quiet') + if address_result.exit_code != 0 { + return error('Failed to get public key: ${address_result.output}') + } + address := address_result.output.trim_space() + + // Get the secret key + show_result := os.execute('stellar keys show ${name} --quiet') + if show_result.exit_code != 0 { + return error('Failed to get secret key: ${show_result.output}') + } + secret := show_result.output.trim_space() + + // Return the StellarAccountKeys struct + return StellarAccountKeys{ + name: name + address: address + secret: secret + } +} + +pub fn get_network_config(network StellarNetwork) !NetworkConfig { + rpc_url, passphrase := match network { + .mainnet { + mainnet_rpc_url, mainnet_passphrase + } + .testnet { + testnet_rpc_url, testnet_passphrase + } + } + return NetworkConfig{ + url: rpc_url + passphrase: passphrase + } +} + +pub fn encode_tx_to_xdr(json_encoding string) !string { + cmd := "echo '${json_encoding}' | stellar xdr encode --type TransactionEnvelope" + result := os.execute(cmd) + if result.exit_code != 0 { + return error('failed to encode tx: ${result.output}') + } + + return result.output.trim_space() +} + +// Struct to hold arguments for creating a Stellar account. +@[params] +pub struct GenerateAccountArgs { +pub mut: + network StellarNetwork = .testnet // Specifies the Stellar network (testnet or mainnet). Defaults to testnet. + name string @[required] // Name of the account. This is required. + fund bool // Whether to fund the account on the test network after creation. + cache bool // Whether to cache the generated keys locally. +} + +// Generates a new Stellar account and returns the associated keys. +// This function generates a new Stellar account using the 'stellar keys generate' command. +// If the 'fund' parameter is true, the account is funded on the network. +// If the 'cache' parameter is true, the generated keys are cached locally. +// +// Arguments: +// - `args` (CreateAccountArgs): Struct containing account creation parameters. +// +// Returns: +// - `StellarAccountKeys`: Struct with the public and secret keys for the account. +// +// Errors: +// - Returns an error if key generation fails or cached key removal fails. +pub fn generate_keys(args GenerateAccountArgs) !StellarAccountKeys { + // Validate the network. + if args.network != .testnet && args.fund { + return error('The fund parameter can only be set to true for the testnet network.') + } + + // Construct the CLI command for generating Stellar keys. + mut cmd := 'stellar keys generate ${args.name} --network ${args.network}' + if args.fund { + cmd += ' --fund' + } else { + cmd += ' --no-fund' + } + + // Execute the command and check for errors. + result := os.execute(cmd) + if result.exit_code != 0 { + return error('Failed to generate keys: ${result.output}') + } + + // Retrieve the generated account keys. + keys := get_account_keys(args.name) or { return error('Failed to get keys: ${err}') } + + // Optionally remove cached keys. + if !args.cache { + remove_cached_keys(name: keys.name) or { + return error('Failed to remove cached keys: ${err}') + } + } + + return keys +} + +// Struct to hold arguments for removing cached Stellar keys. +@[params] +pub struct RemoveCachedKeysArgs { +pub mut: + network StellarNetwork = .testnet // Specifies the Stellar network (testnet or mainnet). Defaults to testnet. + name string @[required] // Name of the account. This is required. +} + +// Removes cached Stellar keys for a specific account. +// +// Arguments: +// - `args` (RemoveCachedKeysArgs): Struct containing parameters for key removal. +// +// Errors: +// - Returns an error if the removal command fails. +fn remove_cached_keys(args RemoveCachedKeysArgs) ! { + cmd := 'stellar keys rm ${args.name}' + result := os.execute(cmd) + if result.exit_code != 0 { + return error('Failed to remove cached keys: ${result.output}') + } +} + +// Funds a Stellar account on the test network using Friendbot. +// +// Arguments: +// - `address` (string): The public key of the account to be funded. +// +// Errors: +// - Returns an error if the funding request fails. +pub fn fund_account(address string) ! { + mut client := httpconnection.new( + name: 'stellar' + url: 'https://friendbot.stellar.org/' + )! + + client.get( + prefix: '?addr=${address}' + )! +} + +// Struct to hold a transaction signer. +@[params] +pub struct NewSignerArgs { +pub mut: + key string @[required] // Signer address + weight int = 1 // Weight +} + +// adding a new signer +pub fn new_signer(args NewSignerArgs) TXSigner { + return TXSigner{ + key: args.key + weight: args.weight + } +} + +pub fn get_offer_price(price f32) Price { + nums := price.str().split('.') + n := int(price * int(math.pow(10, nums[1].len))) + d := int(math.pow(10, nums[1].len)) + return Price{ + n: n + d: d + } +} + +pub fn fetch_highest_bid_price(orderBook OrderBook) Price { + // Parse highest bid price from the order book response + if orderBook.bids.len == 0 { + return Price{ + n: 0 + d: 1 + } + } + + return orderBook.bids[0].price_r +} + +pub fn fetch_lowest_ask_price(orderBook OrderBook) Price { + // Parse highest bid price from the order book response + if orderBook.asks.len == 0 { + return Price{ + n: 0 + d: 1 + } + } + + return orderBook.asks[0].price_r +} + +pub struct MakeOfferResult { +pub mut: + offer_id u64 + claimed bool +} + +pub fn get_offer_id_from_result_xdr(result_xdr string) !MakeOfferResult { + cmd := 'echo ${result_xdr} | stellar xdr decode --type TransactionResult' + tx_result := os.execute(cmd) + if tx_result.exit_code != 0 { + return error('Failed to decode transaction result: ${tx_result.output}') + } + + data := json2.raw_decode(tx_result.output.trim_space())!.as_map() + + offer_id := find_key_recursive(data, 'offer_id')!.u64() + if tx_result.output.contains('"offer":"deleted"') { + return MakeOfferResult{ + offer_id: offer_id + claimed: true + } + } + + return MakeOfferResult{ + offer_id: offer_id + } +} + +fn find_key_recursive(data map[string]json2.Any, key_to_find string) !json2.Any { + for key, value in data { + if key == key_to_find { + return value + } + + if value is map[string]json2.Any { + result := find_key_recursive(value as map[string]json2.Any, key_to_find) or { continue } + return result + } + + if value is []json2.Any { + for item in value { + if item is map[string]json2.Any { + result := find_key_recursive(item as map[string]json2.Any, key_to_find) or { + continue + } + return result + } + } + } + } + + return error('Key ${key_to_find} not found') +} + +fn url_encode(map_ map[string]json2.Any) string { + mut formated := '' + + for k, v in map_ { + if formated != '' { + formated += '&' + k + '=' + v.str() + } else { + formated = k + '=' + v.str() + } + } + return formated +} diff --git a/crystallib/clients/meilisearch/client.v b/crystallib/clients/meilisearch/client.v index a1628b5a4..c3ad2ed49 100644 --- a/crystallib/clients/meilisearch/client.v +++ b/crystallib/clients/meilisearch/client.v @@ -9,7 +9,8 @@ pub fn (mut client MeilisearchClient) health() !Health { req := httpconnection.Request{ prefix: 'health' } - response := client.http.get_json(req)! + mut http := client.httpclient()! + response := http.get_json(req)! return json2.decode[Health](response) } @@ -18,7 +19,8 @@ pub fn (mut client MeilisearchClient) version() !Version { req := httpconnection.Request{ prefix: 'version' } - response := client.http.get_json(req)! + mut http := client.httpclient()! + response := http.get_json(req)! return json2.decode[Version](response) } @@ -29,8 +31,8 @@ pub fn (mut client MeilisearchClient) create_index(args CreateIndexArgs) !Create method: .post data: json2.encode(args) } - - response := client.http.post_json_str(req)! + mut http := client.httpclient()! + response := http.post_json_str(req)! return json2.decode[CreateIndexResponse](response) } @@ -39,7 +41,8 @@ pub fn (mut client MeilisearchClient) get_index(uid string) !GetIndexResponse { req := httpconnection.Request{ prefix: 'indexes/${uid}' } - response := client.http.get_json(req)! + mut http := client.httpclient()! + response := http.get_json(req)! return json2.decode[GetIndexResponse](response) } @@ -48,7 +51,8 @@ pub fn (mut client MeilisearchClient) list_indexes(args ListIndexArgs) ![]GetInd req := httpconnection.Request{ prefix: 'indexes?limit=${args.limit}&offset=${args.offset}' } - response := client.http.get_json(req)! + mut http := client.httpclient()! + response := http.get_json(req)! list_response := json.decode(ListResponse[GetIndexResponse], response)! return list_response.results } @@ -58,7 +62,8 @@ pub fn (mut client MeilisearchClient) delete_index(uid string) !DeleteIndexRespo req := httpconnection.Request{ prefix: 'indexes/${uid}' } - response := client.http.delete(req)! + mut http := client.httpclient()! + response := http.delete(req)! return json2.decode[DeleteIndexResponse](response) } @@ -67,7 +72,8 @@ pub fn (mut client MeilisearchClient) get_settings(uid string) !IndexSettings { req := httpconnection.Request{ prefix: 'indexes/${uid}/settings' } - response := client.http.get_json_dict(req)! + mut http := client.httpclient()! + response := http.get_json_dict(req)! mut settings := IndexSettings{} if ranking_rules := response['rankingRules'] { @@ -102,7 +108,8 @@ pub fn (mut client MeilisearchClient) update_settings(uid string, settings Index method: .patch data: json2.encode(settings) } - return client.http.post_json_str(req) + mut http := client.httpclient()! + return http.post_json_str(req) } // reset_settings resets all settings of an index to default values @@ -111,7 +118,8 @@ pub fn (mut client MeilisearchClient) reset_settings(uid string) !string { prefix: 'indexes/${uid}/settings' method: .delete } - return client.http.delete(req) + mut http := client.httpclient()! + return http.delete(req) } // get_ranking_rules retrieves ranking rules of an index @@ -119,7 +127,8 @@ pub fn (mut client MeilisearchClient) get_ranking_rules(uid string) ![]string { req := httpconnection.Request{ prefix: 'indexes/${uid}/settings/ranking-rules' } - response := client.http.get_json_dict(req)! + mut http := client.httpclient()! + response := http.get_json_dict(req)! return response['rankingRules']!.arr().map(it.str()) } @@ -132,7 +141,8 @@ pub fn (mut client MeilisearchClient) update_ranking_rules(uid string, rules []s 'rankingRules': rules }) } - return client.http.post_json_str(req) + mut http := client.httpclient()! + return http.post_json_str(req) } // reset_ranking_rules resets ranking rules of an index to default values @@ -141,7 +151,8 @@ pub fn (mut client MeilisearchClient) reset_ranking_rules(uid string) !string { prefix: 'indexes/${uid}/settings/ranking-rules' method: .delete } - return client.http.delete(req) + mut http := client.httpclient()! + return http.delete(req) } // get_distinct_attribute retrieves distinct attribute of an index @@ -149,7 +160,8 @@ pub fn (mut client MeilisearchClient) get_distinct_attribute(uid string) !string req := httpconnection.Request{ prefix: 'indexes/${uid}/settings/distinct-attribute' } - response := client.http.get_json_dict(req)! + mut http := client.httpclient()! + response := http.get_json_dict(req)! return response['distinctAttribute']!.str() } @@ -162,7 +174,8 @@ pub fn (mut client MeilisearchClient) update_distinct_attribute(uid string, attr 'distinctAttribute': attribute }) } - return client.http.post_json_str(req) + mut http := client.httpclient()! + return http.post_json_str(req) } // reset_distinct_attribute resets distinct attribute of an index @@ -171,7 +184,8 @@ pub fn (mut client MeilisearchClient) reset_distinct_attribute(uid string) !stri prefix: 'indexes/${uid}/settings/distinct-attribute' method: .delete } - return client.http.delete(req) + mut http := client.httpclient()! + return http.delete(req) } // get_searchable_attributes retrieves searchable attributes of an index @@ -179,7 +193,8 @@ pub fn (mut client MeilisearchClient) get_searchable_attributes(uid string) ![]s req := httpconnection.Request{ prefix: 'indexes/${uid}/settings/searchable-attributes' } - response := client.http.get_json_dict(req)! + mut http := client.httpclient()! + response := http.get_json_dict(req)! return response['searchableAttributes']!.arr().map(it.str()) } @@ -192,7 +207,8 @@ pub fn (mut client MeilisearchClient) update_searchable_attributes(uid string, a 'searchableAttributes': attributes }) } - return client.http.post_json_str(req) + mut http := client.httpclient()! + return http.post_json_str(req) } // reset_searchable_attributes resets searchable attributes of an index @@ -201,7 +217,8 @@ pub fn (mut client MeilisearchClient) reset_searchable_attributes(uid string) !s prefix: 'indexes/${uid}/settings/searchable-attributes' method: .delete } - return client.http.delete(req) + mut http := client.httpclient()! + return http.delete(req) } // get_displayed_attributes retrieves displayed attributes of an index @@ -209,7 +226,8 @@ pub fn (mut client MeilisearchClient) get_displayed_attributes(uid string) ![]st req := httpconnection.Request{ prefix: 'indexes/${uid}/settings/displayed-attributes' } - response := client.http.get_json_dict(req)! + mut http := client.httpclient()! + response := http.get_json_dict(req)! return response['displayedAttributes']!.arr().map(it.str()) } @@ -222,7 +240,8 @@ pub fn (mut client MeilisearchClient) update_displayed_attributes(uid string, at 'displayedAttributes': attributes }) } - return client.http.post_json_str(req) + mut http := client.httpclient()! + return http.post_json_str(req) } // reset_displayed_attributes resets displayed attributes of an index @@ -231,7 +250,8 @@ pub fn (mut client MeilisearchClient) reset_displayed_attributes(uid string) !st prefix: 'indexes/${uid}/settings/displayed-attributes' method: .delete } - return client.http.delete(req) + mut http := client.httpclient()! + return http.delete(req) } // get_stop_words retrieves stop words of an index @@ -239,7 +259,8 @@ pub fn (mut client MeilisearchClient) get_stop_words(uid string) ![]string { req := httpconnection.Request{ prefix: 'indexes/${uid}/settings/stop-words' } - response := client.http.get_json_dict(req)! + mut http := client.httpclient()! + response := http.get_json_dict(req)! return response['stopWords']!.arr().map(it.str()) } @@ -252,7 +273,8 @@ pub fn (mut client MeilisearchClient) update_stop_words(uid string, words []stri 'stopWords': words }) } - return client.http.post_json_str(req) + mut http := client.httpclient()! + return http.post_json_str(req) } // reset_stop_words resets stop words of an index @@ -261,7 +283,8 @@ pub fn (mut client MeilisearchClient) reset_stop_words(uid string) !string { prefix: 'indexes/${uid}/settings/stop-words' method: .delete } - return client.http.delete(req) + mut http := client.httpclient()! + return http.delete(req) } // get_synonyms retrieves synonyms of an index @@ -269,7 +292,8 @@ pub fn (mut client MeilisearchClient) get_synonyms(uid string) !map[string][]str req := httpconnection.Request{ prefix: 'indexes/${uid}/settings/synonyms' } - response := client.http.get_json_dict(req)! + mut http := client.httpclient()! + response := http.get_json_dict(req)! mut synonyms := map[string][]string{} for key, value in response['synonyms']!.as_map() { synonyms[key] = value.arr().map(it.str()) @@ -286,7 +310,8 @@ pub fn (mut client MeilisearchClient) update_synonyms(uid string, synonyms map[s 'synonyms': synonyms }) } - return client.http.post_json_str(req) + mut http := client.httpclient()! + return http.post_json_str(req) } // reset_synonyms resets synonyms of an index @@ -295,7 +320,8 @@ pub fn (mut client MeilisearchClient) reset_synonyms(uid string) !string { prefix: 'indexes/${uid}/settings/synonyms' method: .delete } - return client.http.delete(req) + mut http := client.httpclient()! + return http.delete(req) } // get_filterable_attributes retrieves filterable attributes of an index @@ -303,7 +329,8 @@ pub fn (mut client MeilisearchClient) get_filterable_attributes(uid string) ![]s req := httpconnection.Request{ prefix: 'indexes/${uid}/settings/filterable-attributes' } - response := client.http.get_json_dict(req)! + mut http := client.httpclient()! + response := http.get_json_dict(req)! return response['filterableAttributes']!.arr().map(it.str()) } @@ -312,11 +339,11 @@ pub fn (mut client MeilisearchClient) update_filterable_attributes(uid string, a req := httpconnection.Request{ prefix: 'indexes/${uid}/settings/filterable-attributes' method: .put - data: json2.encode({ - 'filterableAttributes': attributes - }) + data: json.encode(attributes) } - return client.http.post_json_str(req) + mut http := client.httpclient()! + response := http.send(req)! + return response.data } // reset_filterable_attributes resets filterable attributes of an index @@ -325,7 +352,8 @@ pub fn (mut client MeilisearchClient) reset_filterable_attributes(uid string) !s prefix: 'indexes/${uid}/settings/filterable-attributes' method: .delete } - return client.http.delete(req) + mut http := client.httpclient()! + return http.delete(req) } // get_sortable_attributes retrieves sortable attributes of an index @@ -333,7 +361,8 @@ pub fn (mut client MeilisearchClient) get_sortable_attributes(uid string) ![]str req := httpconnection.Request{ prefix: 'indexes/${uid}/settings/sortable-attributes' } - response := client.http.get_json_dict(req)! + mut http := client.httpclient()! + response := http.get_json_dict(req)! return response['sortableAttributes']!.arr().map(it.str()) } @@ -346,7 +375,8 @@ pub fn (mut client MeilisearchClient) update_sortable_attributes(uid string, att 'sortableAttributes': attributes }) } - return client.http.post_json_str(req) + mut http := client.httpclient()! + return http.post_json_str(req) } // reset_sortable_attributes resets sortable attributes of an index @@ -355,7 +385,8 @@ pub fn (mut client MeilisearchClient) reset_sortable_attributes(uid string) !str prefix: 'indexes/${uid}/settings/sortable-attributes' method: .delete } - return client.http.delete(req) + mut http := client.httpclient()! + return http.delete(req) } // get_typo_tolerance retrieves typo tolerance settings of an index @@ -364,7 +395,8 @@ pub fn (mut client MeilisearchClient) get_typo_tolerance(uid string) !TypoTolera prefix: 'indexes/${uid}/settings/typo-tolerance' } - response := client.http.get_json_dict(req)! + mut http := client.httpclient()! + response := http.get_json_dict(req)! min_word_size_for_typos := json2.decode[MinWordSizeForTypos](response['minWordSizeForTypos']!.json_str())! mut typo_tolerance := TypoTolerance{ enabled: response['enabled']!.bool() @@ -388,7 +420,8 @@ pub fn (mut client MeilisearchClient) update_typo_tolerance(uid string, typo_tol method: .patch data: json2.encode(typo_tolerance) } - return client.http.post_json_str(req) + mut http := client.httpclient()! + return http.post_json_str(req) } // reset_typo_tolerance resets typo tolerance settings of an index @@ -397,5 +430,29 @@ pub fn (mut client MeilisearchClient) reset_typo_tolerance(uid string) !string { prefix: 'indexes/${uid}/settings/typo-tolerance' method: .delete } - return client.http.delete(req) + mut http := client.httpclient()! + return http.delete(req) +} + + +@[params] +pub struct EperimentalFeaturesArgs{ +pub mut: + vector_store bool @[json: 'vectorStore'] + metrics bool @[json: 'metrics'] + logs_route bool @[json: 'logsRoute'] + contains_filter bool @[json: 'containsFilter'] + edit_documents_by_function bool @[json: 'editDocumentsByFunction'] } + +pub fn (mut client MeilisearchClient) enable_eperimental_feature(args EperimentalFeaturesArgs) !EperimentalFeaturesArgs { + req := httpconnection.Request{ + prefix: 'experimental-features' + method: .patch, + data: json.encode(args) + } + + mut http := client.httpclient()! + response := http.send(req)! + return json.decode(EperimentalFeaturesArgs, response.data) +} \ No newline at end of file diff --git a/crystallib/clients/meilisearch/document_test.v b/crystallib/clients/meilisearch/document_test.v index 27d716692..c9414f513 100644 --- a/crystallib/clients/meilisearch/document_test.v +++ b/crystallib/clients/meilisearch/document_test.v @@ -11,13 +11,8 @@ pub mut: } // Set up a test client instance -fn setup_client() !MeilisearchClient { - config := ClientConfig{ - host: 'http://localhost:7700' - api_key: 'be61fdce-c5d4-44bc-886b-3a484ff6c531' - } - factory := new_factory(config) - mut client := factory.get()! +fn setup_client() !&MeilisearchClient { + mut client := get()! return client } @@ -200,6 +195,85 @@ fn test_search() { assert doc_.hits[0].id == 3 } +fn test_facet_search() { + mut client := setup_client()! + index_name := rand.string(5) + + documents := [ + MeiliDocument{ + id: 1 + title: 'Life' + content: 'Two men in 1930s Mississippi become friends after being sentenced to life in prison together for a crime they did not commit.' + }, + MeiliDocument{ + id: 2 + title: 'Life' + content: 'In 1955, young photographer Dennis Stock develops a close bond with actor James Dean while shooting pictures of the rising Hollywood star.' + }, + MeiliDocument{ + id: 3 + title: 'Coldplay' + content: 'Coldplay is a british rock band.' + }, + ] + + mut doc := client.add_documents(index_name, documents)! + assert doc.index_uid == index_name + assert doc.type_ == 'documentAdditionOrUpdate' + + time.sleep(500 * time.millisecond) + res := client.update_filterable_attributes(index_name, ['title'])! + + time.sleep(500 * time.millisecond) + settings := client.get_settings(index_name)! + + assert ['title'] == settings.filterable_attributes + + mut doc_ := client.facet_search( + index_name, + facet_name: 'title', + filter: 'title = life' + )! + assert doc_.facet_hits[0].count == 2 +} + +fn test_similar_documents() { + mut client := setup_client()! + index_name := rand.string(5) + + documents := [ + MeiliDocument{ + id: 1 + title: 'Life' + content: 'Two men in 1930s Mississippi become friends after being sentenced to life in prison together for a crime they did not commit.' + }, + MeiliDocument{ + id: 2 + title: 'Life' + content: 'In 1955, young photographer Dennis Stock develops a close bond with actor James Dean while shooting pictures of the rising Hollywood star.' + }, + MeiliDocument{ + id: 3 + title: 'Coldplay' + content: 'Coldplay is a british rock band.' + }, + ] + + mut doc := client.add_documents(index_name, documents)! + assert doc.index_uid == index_name + assert doc.type_ == 'documentAdditionOrUpdate' + + time.sleep(500 * time.millisecond) + + mut doc_ := client.similar_documents( + index_name, + id: 1, + )! + // TODO: Check the meilisearch.SimilarDocumentsResponse error + println('doc_: ${doc_}') + // assert doc_.facet_hits[0].count == 2 +} + // Delete all created indexes fn test_delete_index() { mut client := setup_client()! diff --git a/crystallib/clients/meilisearch/documents.v b/crystallib/clients/meilisearch/documents.v index e95414bb9..01a3b3ba4 100644 --- a/crystallib/clients/meilisearch/documents.v +++ b/crystallib/clients/meilisearch/documents.v @@ -11,8 +11,8 @@ pub fn (mut client MeilisearchClient) add_documents[T](uid string, documents []T method: .post data: json2.encode(documents) } - - response := client.http.post_json_str(req)! + mut http := client.httpclient()! + response := http.post_json_str(req)! return json2.decode[AddDocumentResponse](response)! } @@ -39,7 +39,8 @@ pub fn (mut client MeilisearchClient) get_document[T](args GetDocumentArgs) !T { params: params } - response := client.http.get_json(req)! + mut http := client.httpclient()! + response := http.get_json(req)! return json.decode(T, response) } @@ -64,7 +65,8 @@ pub fn (mut client MeilisearchClient) get_documents[T](uid string, query Documen params: params } - response := client.http.get_json(req)! + mut http := client.httpclient()! + response := http.get_json(req)! decoded := json.decode(ListResponse[T], response)! return decoded.results } @@ -83,7 +85,8 @@ pub fn (mut client MeilisearchClient) delete_document(args DeleteDocumentArgs) ! method: .delete } - response := client.http.delete(req)! + mut http := client.httpclient()! + response := http.delete(req)! return json2.decode[DeleteDocumentResponse](response)! } @@ -94,7 +97,8 @@ pub fn (mut client MeilisearchClient) delete_all_documents(uid string) !DeleteDo method: .delete } - response := client.http.delete(req)! + mut http := client.httpclient()! + response := http.delete(req)! return json2.decode[DeleteDocumentResponse](response)! } @@ -106,7 +110,8 @@ pub fn (mut client MeilisearchClient) update_documents(uid string, documents str data: documents } - response := client.http.post_json_str(req)! + mut http := client.httpclient()! + response := http.post_json_str(req)! return json2.decode[TaskInfo](response)! } @@ -147,6 +152,84 @@ pub fn (mut client MeilisearchClient) search[T](uid string, args SearchArgs) !Se method: .post data: json.encode(args) } - rsponse := client.http.post_json_str(req)! + mut http := client.httpclient()! + rsponse := http.post_json_str(req)! return json.decode(SearchResponse[T], rsponse) } + +@[params] +struct FacetSearchArgs { + facet_name ?string @[json: 'facetName'] // Facet name to search values on + facet_query ?string @[json: 'facetQuery'] // Search query for a given facet value. Defaults to placeholder search if not specified. + q string // Query string + filter ?string // Filter queries by an attribute's value + matching_strategy string = "last" @[json: 'matchingStrategy'] // Strategy used to match query terms within documents + attributes_to_search_on ?[]string @[json: 'attributesToSearchOn'] // Restrict search to the specified attributes +} +@[params] +struct FacetSearchHitsResponse { + value string @[json: 'value'] // Facet value matching the facetQuery + count int @[json: 'count'] // Number of documents with a facet value matching value +} + +@[params] +struct FacetSearchResponse { + facet_hits []FacetSearchHitsResponse @[json: 'facetHits'] // Facet value matching the facetQuery + facet_query string @[json: 'facetQuery'] // The original facetQuery + processing_time_ms int @[json: 'processingTimeMs'] // Processing time of the query +} + +pub fn (mut client MeilisearchClient) facet_search(uid string, args FacetSearchArgs) !FacetSearchResponse { + req := httpconnection.Request{ + prefix: 'indexes/${uid}/facet-search' + method: .post + data: json.encode(args) + } + mut http := client.httpclient()! + rsponse := http.post_json_str(req)! + return json.decode(FacetSearchResponse, rsponse) +} + +@[params] +struct SimilarDocumentsArgs{ + id SimilarDocumentsID @[json: "id"] // Identifier of the target document (mandatory) + embedder string = "default" @[json: "embedder"] // Embedder to use when computing recommendations + attributes_to_retrieve []string = ["*"] @[json: "attributesToRetrieve"] // Attributes to display in the returned documents + offset int @[json: "offset"] // Number of documents to skip + limit int = 20 @[json: "limit"] // Maximum number of documents returned + filter ?string @[json: "filter"] // Filter queries by an attribute's value + show_ranking_score bool @[json: "showRankingScore"] // Display the global ranking score of a document + show_ranking_score_details bool @[json: "showRankingScoreDetails"] // Display detailed ranking score information + ranking_score_threshold ?f64 @[json: "rankingScoreThreshold"] // Exclude results with low ranking scores + retrieve_vectors bool @[json: "retrieveVectors"] // Return document vector data +} + +type SimilarDocumentsID = string | int +@[params] +struct SimilarDocumentsResponse { + hits []SimilarDocumentsHit @[json: 'hits'] // List of hit items + id string @[json: 'id'] // Identifier of the response + processing_time_ms int @[json: 'processingTimeMs'] // Processing time in milliseconds + limit int = 20 @[json: 'limit'] // Maximum number of documents returned + offset int @[json: 'offset'] // Number of documents to skip + estimated_total_hits int @[json: 'estimatedTotalHits'] // Estimated total number of hits +} + +struct SimilarDocumentsHit { + id SimilarDocumentsID @[json: 'id'] // Identifier of the hit item + title string @[json: 'title'] // Title of the hit item +} + + +pub fn (mut client MeilisearchClient) similar_documents(uid string, args SimilarDocumentsArgs) !SimilarDocumentsResponse { + req := httpconnection.Request{ + prefix: 'indexes/${uid}/similar' + method: .post + data: json.encode(args) + } + res := client.enable_eperimental_feature(vector_store: true)! // Enable the feature first. + mut http := client.httpclient()! + rsponse := http.post_json_str(req)! + println('rsponse: ${rsponse}') + return json.decode(SimilarDocumentsResponse, rsponse) +} diff --git a/crystallib/clients/meilisearch/factory.v b/crystallib/clients/meilisearch/factory.v deleted file mode 100644 index ef01738fa..000000000 --- a/crystallib/clients/meilisearch/factory.v +++ /dev/null @@ -1,35 +0,0 @@ -module meilisearch - -import freeflowuniverse.crystallib.clients.httpconnection - -// Factory creates new instances of MeilisearchClient -pub struct Factory { -mut: - config ClientConfig -} - -// new_factory creates a new Factory instance with the given configuration -pub fn new_factory(config ClientConfig) Factory { - return Factory{ - config: config - } -} - -// get returns a new configured MeilisearchClient instance -pub fn (f Factory) get() !MeilisearchClient { - mut http_conn := httpconnection.new( - name: 'meilisearch' - url: f.config.host - retry: f.config.max_retry - )! - - // Add authentication header if API key is provided - if f.config.api_key.len > 0 { - http_conn.default_header.add(.authorization, 'Bearer ${f.config.api_key}') - } - - return MeilisearchClient{ - config: f.config - http: http_conn - } -} diff --git a/crystallib/clients/meilisearch/index_test.v b/crystallib/clients/meilisearch/index_test.v index 43fac6ee4..4ade86394 100755 --- a/crystallib/clients/meilisearch/index_test.v +++ b/crystallib/clients/meilisearch/index_test.v @@ -8,10 +8,8 @@ __global ( ) // Set up a test client instance -fn setup_client() !MeilisearchClient { - //TODO: use he configured entity - factory := new_factory(host:'http://localhost:7700', api_key:'be61fdce-c5d4-44bc-886b-3a484ff6c531') - mut client := factory.get()! +fn setup_client() !&MeilisearchClient { + mut client := get()! return client } diff --git a/crystallib/clients/meilisearch/meilisearch_factory_.v b/crystallib/clients/meilisearch/meilisearch_factory_.v index d785975da..ffcf258b9 100644 --- a/crystallib/clients/meilisearch/meilisearch_factory_.v +++ b/crystallib/clients/meilisearch/meilisearch_factory_.v @@ -102,8 +102,7 @@ pub fn play(args_ PlayArgs) ! { if install_actions.len > 0 { for install_action in install_actions { mut p := install_action.params - mycfg:=cfg_play(p)! - set(mycfg)! + cfg_play(p)! } } diff --git a/crystallib/clients/meilisearch/meilisearch_model.v b/crystallib/clients/meilisearch/meilisearch_model.v index 48821796a..39179089e 100644 --- a/crystallib/clients/meilisearch/meilisearch_model.v +++ b/crystallib/clients/meilisearch/meilisearch_model.v @@ -1,5 +1,6 @@ module meilisearch import freeflowuniverse.crystallib.data.paramsparser +import freeflowuniverse.crystallib.clients.httpconnection import os pub const version = '1.0.0' @@ -25,7 +26,6 @@ pub mut: name string = 'default' api_key string @[secret] host string - httpclient ?httpconnection.HTTPConnection } fn cfg_play(p paramsparser.Params) ! { @@ -36,8 +36,7 @@ fn cfg_play(p paramsparser.Params) ! { api_key: p.get('api_key')! } set(mycfg)! -} - +} fn obj_init(obj_ MeilisearchClient)!MeilisearchClient{ //never call get here, only thing we can do here is work on object itself @@ -46,10 +45,17 @@ fn obj_init(obj_ MeilisearchClient)!MeilisearchClient{ return obj } - -fn (mut self MeilisearchClient) httpclient() !httpconnection.HTTPConnection{ - //todo - panic("implement") +fn (mut self MeilisearchClient) httpclient() !&httpconnection.HTTPConnection{ + mut http_conn := httpconnection.new( + name: 'meilisearch' + url: self.host + )! + + // Add authentication header if API key is provided + if self.api_key.len > 0 { + http_conn.default_header.add(.authorization, 'Bearer ${self.api_key}') + } + return http_conn } diff --git a/crystallib/clients/meilisearch/models.v b/crystallib/clients/meilisearch/models.v index ef6cdcbfd..4089a1629 100644 --- a/crystallib/clients/meilisearch/models.v +++ b/crystallib/clients/meilisearch/models.v @@ -1,15 +1,5 @@ module meilisearch -import freeflowuniverse.crystallib.clients.httpconnection - -// MeilisearchClient is the main client for interacting with Meilisearch -pub struct MeilisearchClient { -pub: - config ClientConfig -mut: - http &httpconnection.HTTPConnection -} - // ClientConfig holds configuration for MeilisearchClient pub struct ClientConfig { pub: diff --git a/crystallib/clients/meilisearch/readme.md b/crystallib/clients/meilisearch/readme.md index 86d2243e0..76e22b0d3 100644 --- a/crystallib/clients/meilisearch/readme.md +++ b/crystallib/clients/meilisearch/readme.md @@ -1,25 +1,58 @@ -# meilisearch +## Meilisearch V Client +This is a simple V client for interacting with a [self-hosted Meilisearch instance](https://www.meilisearch.com/docs/learn/self_hosted/getting_started_with_self_hosted_meilisearch?utm_campaign=oss&utm_medium=home-page&utm_source=docs#setup-and-installation), enabling you to perform operations such as adding, retrieving, deleting, and searching documents within indexes. -To get started +### Getting Started with Self-Hosted Meilisearch -```vlang +To use this V client, ensure you have a **self-hosted Meilisearch instance installed and running**. This quick start will walk you through installing Meilisearch, adding documents, and performing your first search. -import freeflowuniverse.crystallib.clients.meilisearch +#### Requirements + +To follow this setup, you will need `cURL` installed -mut client:= meilisearch.get()! +### Setup and Installation -client... +To install Meilisearch locally, run the following command: +```bash +# Install Meilisearch +curl -L https://install.meilisearch.com | sh ``` -## example heroscript +### Running Meilisearch -```hero -!!meilisearch.configure - secret: '...' - host: 'localhost' - port: 8888 +Start Meilisearch with the following command, replacing `"aSampleMasterKey"` with your preferred master key: + +```bash +# Launch Meilisearch +./meilisearch --master-key="aSampleMasterKey" ``` +--- + +### Running the V Client Tests +This client includes various test cases that demonstrate common operations in Meilisearch, such as creating indexes, adding documents, retrieving documents, deleting documents, and performing searches. To run the tests, you can use the following commands: + +```bash +# Run document-related tests +v -enable-globals -stats crystallib/clients/meilisearch/document_test.v + +# Run index-related tests +v -enable-globals -stats crystallib/clients/meilisearch/index_test.v +``` + +### Example: Getting Meilisearch Server Version + +Here is a quick example of how to retrieve the Meilisearch server version using this V client: + +```v +import freeflowuniverse.crystallib.clients.meilisearch + +fn main() { + mut client := meilisearch.get() or { panic(err) } + version := client.version() or { panic(err) } + println('Meilisearch version: $version') +} +``` +This example connects to your local Meilisearch instance and prints the server version to verify your setup is correct. diff --git a/crystallib/clients/redisclient/redisclient_rpc.v b/crystallib/clients/redisclient/redisclient_rpc.v index 937b392de..4ce197e65 100644 --- a/crystallib/clients/redisclient/redisclient_rpc.v +++ b/crystallib/clients/redisclient/redisclient_rpc.v @@ -73,11 +73,10 @@ pub fn (mut q RedisRpc) result(timeout u64, retqueue string) !string { if r != '' { res := json.decode(Response, r)! if res.error != '' { - return error(res.error) + return res.error } return res.result } - if u64(time.now().unix_milli()) > (start + timeout) { break } @@ -86,9 +85,15 @@ pub fn (mut q RedisRpc) result(timeout u64, retqueue string) !string { return error('timeout on returnqueue: ${retqueue}') } +@[params] +pub struct ProcessParams { +pub: + timeout u64 +} + // to be used by processor, to get request and execute, this is the server side of a RPC mechanism // 2nd argument is a function which needs to execute the job: fn (string,string) !string -pub fn (mut q RedisRpc) process(timeout u64, op fn (string, string) !string) !string { +pub fn (mut q RedisRpc) process(op fn (string, string) !string, params ProcessParams) !string { start := u64(time.now().unix_milli()) for { r := q.redis.rpop(q.key) or { '' } @@ -117,10 +122,10 @@ pub fn (mut q RedisRpc) process(timeout u64, op fn (string, string) !string) !st q.redis.lpush(returnqueue, encoded)! return returnqueue } - if u64(time.now().unix_milli()) > (start + timeout) { + if (params.timeout != 0) && u64(time.now().unix_milli()) > (start + params.timeout) { break } - time.sleep(time.microsecond) + time.sleep(time.millisecond) } return error('timeout for waiting for cmd on ${q.key}') } diff --git a/crystallib/core/base/baseconfig.v b/crystallib/core/base/baseconfig.v index cffbfcd91..e057ecd9b 100644 --- a/crystallib/core/base/baseconfig.v +++ b/crystallib/core/base/baseconfig.v @@ -33,16 +33,13 @@ pub fn (mut self BaseConfig[T]) session() !&Session { // management class of the configs of this obj pub fn (mut self BaseConfig[T]) configurator() !&Configurator[T] { - mut configurator := self.configurator_ or { - // session := self.session_ or { return error('base config must be initialized') } + if self.configurator_ == none { mut c := configurator_new[T]( instance: self.instance )! self.configurator_ = c - self.configurator_ or { panic('s') } } - - return &configurator + return &(self.configurator_ or { return error('configurator not initialized') }) } // will overwrite the config @@ -56,7 +53,7 @@ pub fn (mut self BaseConfig[T]) config_new() !&T { mut configurator := self.configurator()! mut c := configurator.new()! self.config_ = &c - self.config_ or { panic('s') } + &c } self.config_save()! diff --git a/crystallib/core/base/context.v b/crystallib/core/base/context.v index 2f65c817a..888721423 100644 --- a/crystallib/core/base/context.v +++ b/crystallib/core/base/context.v @@ -63,40 +63,12 @@ pub fn (self Context) guid() string { return '${self.id()}:${self.name()}' } -//////DATA - -// pub fn (mut self Context) str() string { -// return self.heroscript() or { "BUG: can't represent the object properly, I try raw" } -// } - -// fn (mut self Context) str2() string { -// panic("implement") -// //return 'cid:${self.cid} name:${self.name}' -// } - -// pub fn (mut self Context) heroscript() !string { -// panic("implement") -// mut out := '!!core.context_define ${self.str2()}\n' -// mut p:=self.params()! -// if !p.empty() { -// out += '\n!!core.params_context_set' -// out += texttools.indent(p.heroscript(), ' ') + '\n' -// } -// // if self.snippets.len > 0 { -// // for key, snippet in self.snippets { -// // out += '\n!!core.snippet guid:${self.guid()} name:${key}' -// // out += texttools.indent(snippet.heroscript()," ") + '\n' -// // } -// // } -// return out -// } - pub fn (mut self Context) redis() !&redisclient.Redis { mut r2 := self.redis_ or { mut r := redisclient.core_get()! if self.config.id > 0 { // make sure we are on the right db - r.selectdb(self.config.id)! + r.selectdb(int(self.config.id))! } self.redis_ = &r &r @@ -177,40 +149,6 @@ pub fn (mut self Context) hero_config_get(cat string, name string) !string { return config_file.read()! } -/////////////PRIVKEY - -// pub fn (mut self Context) privkey_new() !&secp256k1.Secp256k1 { -// mypk := secp256k1.new()! -// return self.privkey_set(mypk.private_key_hex())! -// } - -// pub fn (mut self Context) privkey_set(keyhex string) !&secp256k1.Secp256k1 { -// privkeyencr := self.secret_encrypt(keyhex)! -// self.config.priv_key = privkeyencr -// // self.save()! -// return self.privkey() -// } - -// // get the private key -// pub fn (mut self Context) privkey() !&secp256k1.Secp256k1 { -// mut mypk := self.priv_key_ or { -// mut r := self.redis()! -// mut key := r.get('context:privkey') or { '' } -// if key == '' { -// return error("can't find priv key for context:${self.config.id}") -// } -// key = self.secret_decrypt(key)! -// mut mypk := secp256k1.new( -// privhex: key -// )! -// self.priv_key_ = &mypk -// &mypk -// } - -// return mypk -// } - -// will use our secret as configured for the hero to encrypt, uses base64 pub fn (mut self Context) secret_encrypt(txt string) !string { return aes_symmetric.encrypt_str(txt, self.secret_get()!) } @@ -232,8 +170,6 @@ pub fn (mut self Context) secret_get() !string { return secret } -/////////////SECRET MANAGEMENT - // show a UI in console to configure the secret pub fn (mut self Context) secret_configure() ! { mut myui := ui.new()! diff --git a/crystallib/core/codemodel/file.v b/crystallib/core/codemodel/file.v new file mode 100644 index 000000000..dfc25208a --- /dev/null +++ b/crystallib/core/codemodel/file.v @@ -0,0 +1,19 @@ +module codemodel + +import freeflowuniverse.crystallib.core.pathlib + +pub interface IFile { + write(string, WriteOptions) ! +} + +pub struct File { +pub mut: + name string + extension string + content string +} + +pub fn (f File) write(path string, params WriteOptions) ! { + mut fd_file := pathlib.get_file(path: '${path}/${f.name}.${f.extension}')! + fd_file.write(f.content)! +} \ No newline at end of file diff --git a/crystallib/core/codemodel/folder.v b/crystallib/core/codemodel/folder.v new file mode 100644 index 000000000..077f92065 --- /dev/null +++ b/crystallib/core/codemodel/folder.v @@ -0,0 +1,30 @@ +module codemodel + +import freeflowuniverse.crystallib.core.pathlib + +pub interface IFolder { + name string + files []IFile + write(string, WriteOptions) ! +} + +pub struct Folder { +pub: + name string + files []IFile +} + +pub fn (f Folder) write(path string, options WriteOptions) ! { + mut dir := pathlib.get_dir( + path: '${path}/${f.name}' + empty: options.overwrite + )! + + if !options.overwrite && dir.exists() { + return + } + + for file in f.files { + file.write(dir.path, options)! + } +} \ No newline at end of file diff --git a/crystallib/core/codemodel/function.v b/crystallib/core/codemodel/function.v new file mode 100644 index 000000000..1f5576be3 --- /dev/null +++ b/crystallib/core/codemodel/function.v @@ -0,0 +1,80 @@ +module codemodel + +pub struct Function { +pub: + name string @[omitempty] + receiver Param @[omitempty] + is_pub bool @[omitempty] + mod string @[omitempty] +pub mut: + description string @[omitempty] + params []Param @[omitempty] + body string @[omitempty] + result Result @[omitempty] + has_return bool @[omitempty] +} + +pub struct Param { +pub: + required bool @[omitempty] + mutable bool @[omitempty] + is_shared bool @[omitempty] + is_optional bool @[omitempty] + description string @[omitempty] + name string @[omitempty] + typ Type @[omitempty] + struct_ Struct @[omitempty] +} + +pub struct Result { +pub mut: + typ Type @[omitempty] + description string @[omitempty] + name string @[omitempty] + result bool @[omitempty] // whether is result type + optional bool @[omitempty] // whether is result type + structure Struct @[omitempty] +} + +pub fn parse_function(code_ string) !Function { + mut code := code_.trim_space() + is_pub := code.starts_with('pub ') + if is_pub { + code = code.trim_string_left('pub ').trim_space() + } + + is_fn := code.starts_with('fn ') + if !is_fn { + return error('invalid function format') + } + code = code.trim_string_left('fn ').trim_space() + + receiver := if code.starts_with('(') { + param_str := code.all_after('(').all_before(')').trim_space() + code = code.all_after(')').trim_space() + parse_param(param_str)! + } else { + Param{} + } + + name := code.all_before('(').trim_space() + code = code.trim_string_left(name).trim_space() + + params_str := code.all_after('(').all_before(')') + params := if params_str.trim_space() != '' { + params_str_lst := params_str.split(',') + params_str_lst.map(parse_param(it)!) + } else { + []Param{} + } + result := parse_result(code.all_after(')').all_before('{').replace(' ', ''))! + + body := if code.contains('{') { code.all_after('{').all_before_last('}') } else { '' } + return Function{ + name: name + receiver: receiver + params: params + result: result + body: body + } +} \ No newline at end of file diff --git a/crystallib/core/codemodel/model.v b/crystallib/core/codemodel/model.v index f33510535..a3d10ec74 100644 --- a/crystallib/core/codemodel/model.v +++ b/crystallib/core/codemodel/model.v @@ -59,63 +59,6 @@ pub: arg string // [name: arg] } -pub struct Function { -pub: - name string - receiver Param - is_pub bool - mod string -pub mut: - description string - params []Param - body string - result Result - has_return bool -} - -pub fn parse_function(code_ string) !Function { - mut code := code_.trim_space() - is_pub := code.starts_with('pub ') - if is_pub { - code = code.trim_string_left('pub ').trim_space() - } - - is_fn := code.starts_with('fn ') - if !is_fn { - return error('invalid function format') - } - code = code.trim_string_left('fn ').trim_space() - - receiver := if code.starts_with('(') { - param_str := code.all_after('(').all_before(')').trim_space() - code = code.all_after(')').trim_space() - parse_param(param_str)! - } else { - Param{} - } - - name := code.all_before('(').trim_space() - code = code.trim_string_left(name).trim_space() - - params_str := code.all_after('(').all_before(')') - params := if params_str.trim_space() != '' { - params_str_lst := params_str.split(',') - params_str_lst.map(parse_param(it)!) - } else { - []Param{} - } - result := parse_result(code.all_after(')').all_before('{').replace(' ', ''))! - - body := if code.contains('{') { code.all_after('{').all_before_last('}') } else { '' } - return Function{ - name: name - receiver: receiver - params: params - result: result - body: body - } -} - pub fn parse_param(code_ string) !Param { mut code := code_.trim_space() is_mut := code.starts_with('mut ') @@ -149,28 +92,6 @@ pub fn parse_result(code_ string) !Result { } } -pub struct Param { -pub: - required bool - mutable bool - is_shared bool - is_optional bool - description string - name string - typ Type - struct_ Struct -} - -pub struct Result { -pub mut: - typ Type - description string - name string - result bool // whether is result type - optional bool // whether is result type - structure Struct -} - // todo: maybe make 'is_' fields methods? pub struct Type { pub mut: @@ -185,18 +106,6 @@ pub mut: mod string @[str: skip] } -pub struct File { -pub mut: - name string - extension string - content string -} - -pub fn (f File) write(path string) ! { - mut fd_file := pathlib.get_file(path: '${path}/${f.name}.${f.extension}')! - fd_file.write(f.content)! -} - pub struct Alias { pub: name string diff --git a/crystallib/core/codemodel/module.v b/crystallib/core/codemodel/module.v index 660dede5d..d91a86abf 100644 --- a/crystallib/core/codemodel/module.v +++ b/crystallib/core/codemodel/module.v @@ -6,13 +6,25 @@ import os pub struct Module { pub mut: name string - files []CodeFile - misc_files []File - // model CodeFile - // methods CodeFile + files []IFile + folders []IFolder + // model VFile + // methods VFile } -pub fn (mod Module) write_v(path string, options WriteOptions) ! { +pub fn new_module(mod Module) Module { + return Module { + ...mod + files: mod.files.map( + if it is VFile { + IFile(VFile{...it, mod: mod.name}) + } else {it} + ) + } +} + + +pub fn (mod Module) write(path string, options WriteOptions) ! { mut module_dir := pathlib.get_dir( path: '${path}/${mod.name}' empty: options.overwrite @@ -23,11 +35,11 @@ pub fn (mod Module) write_v(path string, options WriteOptions) ! { } for file in mod.files { - file.write_v(module_dir.path, options)! - } - for file in mod.misc_files { - file.write(module_dir.path)! + file.write(module_dir.path, options)! } + // for file in mod.misc_files { + // file.write(module_dir.path)! + // } if options.format { os.execute('v fmt -w ${module_dir.path}') diff --git a/crystallib/core/codemodel/codefile.v b/crystallib/core/codemodel/vfile.v similarity index 81% rename from crystallib/core/codemodel/codefile.v rename to crystallib/core/codemodel/vfile.v index dfcbaacf3..bad88337c 100644 --- a/crystallib/core/codemodel/codefile.v +++ b/crystallib/core/codemodel/vfile.v @@ -4,7 +4,8 @@ import freeflowuniverse.crystallib.core.texttools import freeflowuniverse.crystallib.core.pathlib import os -pub struct CodeFile { + +pub struct VFile { pub mut: name string mod string @@ -14,15 +15,15 @@ pub mut: content string } -pub fn new_file(config CodeFile) CodeFile { - return CodeFile{ +pub fn new_file(config VFile) VFile { + return VFile{ ...config mod: texttools.name_fix(config.mod) items: config.items } } -pub fn (mut file CodeFile) add_import(import_ Import) ! { +pub fn (mut file VFile) add_import(import_ Import) ! { for mut i in file.imports { if i.mod == import_.mod { i.add_types(import_.types) @@ -32,7 +33,7 @@ pub fn (mut file CodeFile) add_import(import_ Import) ! { file.imports << import_ } -pub fn (code CodeFile) write_v(path string, options WriteOptions) ! { +pub fn (code VFile) write(path string, options WriteOptions) ! { filename := '${options.prefix}${texttools.name_fix(code.name)}.v' mut filepath := pathlib.get('${path}/${filename}') @@ -67,7 +68,7 @@ pub fn (code CodeFile) write_v(path string, options WriteOptions) ! { } } -pub fn (file CodeFile) get_function(name string) ?Function { +pub fn (file VFile) get_function(name string) ?Function { functions := file.items.filter(it is Function).map(it as Function) target_lst := functions.filter(it.name == name) @@ -80,7 +81,7 @@ pub fn (file CodeFile) get_function(name string) ?Function { return target_lst[0] } -pub fn (mut file CodeFile) set_function(function Function) ! { +pub fn (mut file VFile) set_function(function Function) ! { function_names := file.items.map(if it is Function { it.name } else { '' }) index := function_names.index(function.name) @@ -90,10 +91,10 @@ pub fn (mut file CodeFile) set_function(function Function) ! { file.items[index] = function } -pub fn (file CodeFile) functions() []Function { +pub fn (file VFile) functions() []Function { return file.items.filter(it is Function).map(it as Function) } -pub fn (file CodeFile) structs() []Struct { +pub fn (file VFile) structs() []Struct { return file.items.filter(it is Struct).map(it as Struct) } \ No newline at end of file diff --git a/crystallib/core/codemodel/vgen.v b/crystallib/core/codemodel/vgen.v index e6532f93d..ef8fb524b 100644 --- a/crystallib/core/codemodel/vgen.v +++ b/crystallib/core/codemodel/vgen.v @@ -182,7 +182,7 @@ pub fn (param Param) vgen() string { if param.mutable { vstr = 'mut ${vstr}' } - return '(${vstr})' + return '${vstr}' } // vgen_function generates a function statement for a function diff --git a/crystallib/core/codeparser/vparser.v b/crystallib/core/codeparser/vparser.v index 1cecc738f..836cf97b0 100644 --- a/crystallib/core/codeparser/vparser.v +++ b/crystallib/core/codeparser/vparser.v @@ -4,7 +4,7 @@ import v.ast import v.parser import freeflowuniverse.crystallib.core.pathlib import freeflowuniverse.crystallib.ui.console -import freeflowuniverse.crystallib.core.codemodel { Module, CodeFile, CodeItem, Function, Import, Param, Result, Struct, StructField, Sumtype, Type, parse_consts, parse_import } +import freeflowuniverse.crystallib.core.codemodel {IFile, Module, VFile, CodeItem, Function, Import, Param, Result, Struct, StructField, Sumtype, Type, parse_consts, parse_import } import v.pref // VParser holds configuration of parsing @@ -81,11 +81,11 @@ fn (vparser VParser) parse_vpath(mut path pathlib.Path, mut table ast.Table) ![] } // parse_vfile parses and returns code items from a v code file -pub fn parse_file(path string, vparser VParser) !CodeFile { +pub fn parse_file(path string, vparser VParser) !VFile { mut file := pathlib.get_file(path: path)! mut table := ast.new_table() items := vparser.parse_vfile(file.path, mut table) - return CodeFile{ + return VFile{ name: file.name().trim_string_right('.v') imports: parse_imports(file.read()!) consts: parse_consts(file.read()!)! @@ -192,7 +192,7 @@ pub fn parse_module(path_ string, vparser VParser) !Module { } mut table := ast.new_table() - mut code := []CodeFile{} + mut code := []IFile{} // fpref := &pref.Preferences{ // preferences for parsing // is_fmt: true // } diff --git a/crystallib/core/herocmds/mdbook.v b/crystallib/core/herocmds/mdbook.v index 2160e073f..2b1d305b0 100644 --- a/crystallib/core/herocmds/mdbook.v +++ b/crystallib/core/herocmds/mdbook.v @@ -1,7 +1,9 @@ module herocmds import freeflowuniverse.crystallib.web.mdbook +import freeflowuniverse.crystallib.core.pathlib import cli { Command, Flag } +import os import freeflowuniverse.crystallib.ui.console // path string //if location on filessytem, if exists, this has prio on git_url @@ -52,9 +54,31 @@ If you do -gr it will pull newest book content from git and overwrite local chan description: 'will open the generated book.' }) + + mut cmd_list := Command{ + sort_flags: true + name: 'list' + execute: cmd_mdbook_list + description: 'will list existing mdbooks' + } + + cmd_mdbook.add_command(cmd_list) cmdroot.add_command(cmd_mdbook) } +fn cmd_mdbook_list(cmd Command) ! { + console.print_header('MDBooks:') + build_path := os.join_path(os.home_dir(), 'hero/var/mdbuild') + mut build_dir := pathlib.get_dir(path: build_path)! + list := build_dir.list( + recursive: false + dirs_only: true + )! + for path in list.paths { + console.print_stdout(path.name()) + } +} + fn cmd_mdbook_execute(cmd Command) ! { mut name := cmd.flags.get_string('name') or { '' } @@ -65,7 +89,7 @@ fn cmd_mdbook_execute(cmd Command) ! { mut plbook, _ := plbook_run(cmd)! // get name from the book.generate action if name == '' { - mut a := plbook.action_get(actor: 'mdbook', name: 'export')! + mut a := plbook.action_get(actor: 'mdbook', name: 'define')! name = a.params.get('name') or { '' } } } else { diff --git a/crystallib/core/herocmds/playbook_lib.v b/crystallib/core/herocmds/playbook_lib.v index f0f6fd00e..c24797698 100644 --- a/crystallib/core/herocmds/playbook_lib.v +++ b/crystallib/core/herocmds/playbook_lib.v @@ -104,7 +104,7 @@ pub fn cmd_run_add_flags(mut cmd_run Command) { }) } -// returns the path of the fetched repo +// returns the path of the fetched repo url pub fn plbook_code_get(cmd Command) !string { mut path := cmd.flags.get_string('path') or { '' } mut url := cmd.flags.get_string('url') or { '' } @@ -119,8 +119,6 @@ pub fn plbook_code_get(cmd Command) !string { if coderoot.len > 0 { base.context_new(coderoot: coderoot)! - - // panic('coderoot >0 not supported yet, not imeplemented.') } reset := cmd.flags.get_bool('gitreset') or { false } @@ -133,19 +131,19 @@ pub fn plbook_code_get(cmd Command) !string { pull: pull reset: reset url: url - reload: true + // QUESTION: why should reload be default true? + // reload: true )! - path = repo.get_path()! + path = repo.get_path_of_url(url)! } return path } // same as session_run_get but will also run the playbook -fn plbook_run(cmd Command) !(&playbook.PlayBook, string) { +pub fn plbook_run(cmd Command) !(&playbook.PlayBook, string) { path := plbook_code_get(cmd)! - if path.len == 0 { return error(cmd.help_message()) } diff --git a/crystallib/core/openapi/gen/README.md b/crystallib/core/openapi/gen/README.md deleted file mode 100644 index 84d7c3883..000000000 --- a/crystallib/core/openapi/gen/README.md +++ /dev/null @@ -1,49 +0,0 @@ -## OpenAPI Code Generation Module - - -### Way structure definitions are written and arranged - -Object schemas are defined in an OpenAPI Specification, which define the structure of data passed as parameters to a API Call, and data returned by the calls. - -These schemas therefore require data structures that need to be defined as V `struct`s in code. - -Object schemas defined in the components field of the OpenAPI Specification are assumed to be 'common' to API calls defined in the specification. The `struct`s representing these common object schemas are therefore defined in a `model.v` file. - -After that, the schemas defined in the path operations are generated alongside the Client API Methods they belong to. - -`openapi.json` -```json -{ - "components": { - "schemas": { - "Person": {} - } - }, - "paths": { - "/new_person": { - "post": { - "parameters": [ - { - "name": "person_args", - "schema": { - "type": "object" - } - } - ] - } - } - } -} -``` - -`model.v` -``` -struct Person{} -``` - -`methods.v` -``` -struct NewPersonArgs {} - -fn new_person(person_args NewPersonArgs) Person {} -``` \ No newline at end of file diff --git a/crystallib/core/openapi/model.v b/crystallib/core/openapi/model.v deleted file mode 100644 index d9d1fc002..000000000 --- a/crystallib/core/openapi/model.v +++ /dev/null @@ -1,220 +0,0 @@ -module openapi - -import freeflowuniverse.crystallib.core.openrpc -import freeflowuniverse.crystallib.data.jsonschema - -// todo: report bug: when comps is optional, doesnt work -pub struct OpenAPI { -pub: - openapi string @[required] // This string MUST be the version number of the OpenAPI Specification that the OpenAPI document uses. The openapi field SHOULD be used by tooling to interpret the OpenAPI document. This is not related to the API info.version string. - info Info @[required] // Provides metadata about the API. The metadata MAY be used by tooling as required. - json_schema_dialect ?string // The default value for the $schema keyword within Schema Objects contained within this OAS document. This MUST be in the form of a URI. - servers ?[]Server // An array of Server Objects, which provide connectivity information to a target server. If the servers property is not provided, or is an empty array, the default value would be a Server Object with a url value of /. - paths map[string]PathItem // The available paths and operations for the API. - webhooks ?map[string]PathRef // The incoming webhooks that MAY be received as part of this API and that the API consumer MAY choose to implement. Closely related to the callbacks feature, this section describes requests initiated other than by an API call, for example by an out of band registration. The key name is a unique string to refer to each webhook, while the (optionally referenced) Path Item Object describes a request that may be initiated by the API provider and the expected responses. An example is available. - components Components // An element to hold various schemas for the document. - security ?[]SecurityRequirement // A declaration of which security mechanisms can be used across the API. The list of values includes alternative security requirement objects that can be used. Only one of the security requirement objects need to be satisfied to authorize a request. Individual operations can override this definition. To make security optional, an empty security requirement ({}) can be included in the array. - tags ?[]Tag // A list of tags used by the document with additional metadata. The order of the tags can be used to reflect on their order by the parsing tools. Not all tags that are used by the Operation Object must be declared. The tags that are not declared MAY be organized randomly or based on the tools’ logic. Each tag name in the list MUST be unique. - external_docs ?ExternalDocumentation // Additional external documentation. -} - -pub fn (spec OpenAPI) plain() string { - return '${spec}'.split('\n').filter(!it.contains('Option(none)')).join('\n') -} - -// ``` -// { -// "title": "Sample Pet Store App", -// "summary": "A pet store manager.", -// "description": "This is a sample server for a pet store.", -// "termsOfService": "https://example.com/terms/", -// "contact": { -// "name": "API Support", -// "url": "https://www.example.com/support", -// "email": "support@example.com" -// }, -// "license": { -// "name": "Apache 2.0", -// "url": "https://www.apache.org/licenses/LICENSE-2.0.html" -// }, -// "version": "1.0.1" -// } -// ``` -// The object provides metadata about the API. The metadata MAY be used by the clients if needed, and MAY be presented in editing or documentation generation tools for convenience. -pub struct Info { - title string @[required] // The title of the API - summary string // A short summary of the API. - description string // A description of the API. CommonMark syntax MAY be used for rich text representation. - terms_of_service string // A URL to the Terms of Service for the API. This MUST be in the form of a URL. - contact Contact // The contact information for the exposed API. - license License // The license information for the exposed API. - version string @[required] // The version of the OpenAPI document (which is distinct from the OpenAPI Specification version or the API implementation version). -} - -// ```{ -// "name": "API Support", -// "url": "https://www.example.com/support", -// "email": "support@example.com" -// }``` -// Contact information for the exposed API. -pub struct Contact { - name string // The identifying name of the contact person/organization. - url string // The URL pointing to the contact information. This MUST be in the form of a URL. - email string // The email address of the contact person/organization. This MUST be in the form of an email address. -} - -// ```{ -// "name": "Apache 2.0", -// "identifier": "Apache-2.0" -// }``` -// License information for the exposed API. -pub struct License { - name string @[required] // The license name used for the API. - identifier string // An SPDX license expression for the API. The identifier field is mutually exclusive of the url field. - url string // A URL to the license used for the API. This MUST be in the form of a URL. The url field is mutually exclusive of the identifier field. -} - -// ```{ -// "url": "https://development.gigantic-server.com/v1", -// "description": "Development server" -// }``` -pub struct Server { - url string @[required] // A URL to the target host. This URL supports Server Variables and MAY be relative, to indicate that the host location is relative to the location where the OpenAPI document is being served. Variable substitutions will be made when a variable is named in {brackets}. - description string // An optional string describing the host designated by the URL. CommonMark syntax MAY be used for rich text representation. - variables map[string]openrpc.ServerVariable // A map between a variable name and its value. The value is used for substitution in the server’s URL template. -} - -pub struct Path {} - -pub struct Reference { - ref string @[json: 'ref'; required] // The reference identifier. This MUST be in the form of a URI. - summary string // A short summary which by default SHOULD override that of the referenced component. If the referenced object-type does not allow a summary field, then this field has no effect. - description string // A description which by default SHOULD override that of the referenced component. CommonMark syntax MAY be used for rich text representation. If the referenced object-type does not allow a description field, then this field has no effect. -} - -type PathRef = Path | Reference - -pub struct Components { - schemas map[string]Schema // An object to hold reusable Schema Objects. - responses map[string]ResponseRef // An object to hold reusable Response Objects. - parameters map[string]ParameterRef // An object to hold reusable Parameter Objects. - examples map[string]ExampleRef // An object to hold reusable Example Objects. - request_bodies map[string]RequestBodyRef // An object to hold reusable Request Body Objects. - headers map[string]HeaderRef // An object to hold reusable Header Objects. - security_schemes map[string]SecuritySchemeRef // An object to hold reusable Security Scheme Objects. - links map[string]LinkRef // An object to hold reusable Link Objects. - callbacks map[string]CallbackRef // An object to hold reusable Callback Objects. - path_items map[string]PathItemRef // An object to hold reusable Path Item Object. -} - -pub struct Schema { - type_ string @[json: 'type'] - description string - enum_ []string @[json: 'enum'] - properties map[string]Schema - format string - ref string @[json: '\$ref'] - example string - nullable bool - required ?[]string -} - -type ResponseRef = Reference | Response -type ParameterRef = Parameter | Reference -type SecuritySchemeRef = Reference | SecurityScheme -type ExampleRef = Example | Reference -type RequestBodyRef = Reference | RequestBody -type HeaderRef = Header | Reference -type LinkRef = Link | Reference -type CallbackRef = Callback | Reference -type PathItemRef = PathItem | Reference -type RequestRef = Reference | Request - -pub struct PathItem { - ref ?string // Allows for a referenced definition of this path item. The referenced structure MUST be in the form of a Path Item Object. In case a Path Item Object field appears both in the defined object and the referenced object, the behavior is undefined. See the rules for resolving Relative References. - summary ?string // An optional, string summary, intended to apply to all operations in this path. - description ?string // An optional, string description, intended to apply to all operations in this path. CommonMark syntax MAY be used for rich text representation. - get ?Operation // A definition of a GET operation on this path. - put ?Operation // A definition of a PUT operation on this path. - post ?Operation // A definition of a POST operation on this path. - delete ?Operation // A definition of a DELETE operation on this path. - options ?Operation // A definition of a OPTIONS operation on this path. - head ?Operation // A definition of a HEAD operation on this path. - patch ?Operation // A definition of a PATCH operation on this path. - trace ?Operation // A definition of a TRACE operation on this path. - servers ?[]Server // An alternative server array to service all operations in this path. - parameters ?[]Parameter // A list of parameters that are applicable for all the operations described under this path. These parameters can be overridden at the operation level, but cannot be removed there. The list MUST NOT include duplicated parameters. A unique parameter is defined by a combination of a name and location. The list can use the Reference Object to link to parameters that are defined at the OpenAPI Object’s components/parameters. -} - -pub struct Operation { - tags []string // A list of tags for API documentation control. Tags can be used for logical grouping of operations by resources or any other qualifier. - summary string // A short summary of what the operation does. - description string // A verbose explanation of the operation behavior. CommonMark syntax MAY be used for rich text representation. - external_docs ExternalDocumentation @[json: 'externalDocs'] // Additional external documentation for this operation. - operation_id string @[json: 'operationId'] // Unique string used to identify the operation. The id MUST be unique among all operations described in the API. The operationId value is case-sensitive. Tools and libraries MAY use the operationId to uniquely identify an operation, therefore, it is RECOMMENDED to follow common programming naming conventions. - parameters []Parameter // A list of parameters that are applicable for this operation. If a parameter is already defined at the Path Item, the new definition will override it but can never remove it. The list MUST NOT include duplicated parameters. A unique parameter is defined by a combination of a name and location. The list can use the Reference Object to link to parameters that are defined at the OpenAPI Object’s components/parameters. - request_body RequestRef @[json: 'requestBody'] // The request body applicable for this operation. The requestBody is fully supported in HTTP methods where the HTTP 1.1 specification [RFC7231] has explicitly defined semantics for request bodies. In other cases where the HTTP spec is vague (such as GET, HEAD and DELETE), requestBody is permitted but does not have well-defined semantics and SHOULD be avoided if possible. - responses map[string]Response // The list of possible responses as they are returned from executing this operation. - callbacks map[string]CallbackRef // A map of possible out-of band callbacks related to the parent operation. The key is a unique identifier for the Callback Object. Each value in the map is a Callback Object that describes a request that may be initiated by the API provider and the expected responses. - deprecated bool // Declares this operation to be deprecated. Consumers SHOULD refrain from usage of the declared operation. Default value is false. - security []SecurityRequirement // A declaration of which security mechanisms can be used for this operation. The list of values includes alternative security requirement objects that can be used. Only one of the security requirement objects need to be satisfied to authorize a request. To make security optional, an empty security requirement ({}) can be included in the array. This definition overrides any declared top-level security. To remove a top-level security declaration, an empty array can be used. - servers []Server // An alternative server array to service this operation. If an alternative server object is specified at the Path Item Object or Root level, it will be overridden by this value. -} - -// TODO: currently using map[string]Response -pub struct Responses { - default ResponseRef -} - -pub struct Callback {} - -pub struct Link {} - -pub struct Header {} - -pub struct Request {} - -pub struct Response { - description string @[required] // A description of the response. CommonMark syntax MAY be used for rich text representation. - headers ?map[string]HeaderRef // Maps a header name to its definition. [RFC7230] states header names are case insensitive. If a response header is defined with the name "Content-Type", it SHALL be ignored. - content map[string]MediaType // A map containing descriptions of potential response payloads. The key is a media type or media type range and the value describes it. For responses that match multiple keys, only the most specific key is applicable. e.g. text/plain overrides text/* - links map[string]LinkRef // A map of operations links that can be followed from the response. The key of the map is a short name for the link, following the naming constraints of the names for Component Objects. -} - -// TODO: media type example any field -pub struct MediaType { - schema Schema // The schema defining the content of the request, response, or parameter. - example string // Example of the media type. The example object SHOULD be in the correct format as specified by the media type. The example field is mutually exclusive of the examples field. Furthermore, if referencing a schema which contains an example, the example value SHALL override the example provided by the schema. - examples map[string]ExampleRef // Examples of the media type. Each example object SHOULD match the media type and specified schema if present. The examples field is mutually exclusive of the example field. Furthermore, if referencing a schema which contains an example, the examples value SHALL override the example provided by the schema. - encoding map[string]Encoding // A map between a property name and its encoding information. The key, being the property name, MUST exist in the schema as a property. The encoding object SHALL only apply to requestBody objects when the media type is multipart or application/x-www-form-urlencoded. -} - -pub struct Encoding { - content_type string @[json: 'contentType'] // The Content-Type for encoding a specific property. Default value depends on the property type: for object - application/json; for array – the default is defined based on the inner type; for all other cases the default is application/octet-stream. The value can be a specific media type (e.g. application/json), a wildcard media type (e.g. image/*), or a comma-separated list of the two types. - headers map[string]HeaderRef // A map allowing additional information to be provided as headers, for example Content-Disposition. Content-Type is described separately and SHALL be ignored in this section. This property SHALL be ignored if the request body media type is not a multipart. - style string // Describes how a specific property value will be serialized depending on its type. See Parameter Object for details on the style property. The behavior follows the same values as query parameters, including default values. This property SHALL be ignored if the request body media type is not application/x-www-form-urlencoded or multipart/form-data. If a value is explicitly defined, then the value of contentType (implicit or explicit) SHALL be ignored. - explode bool // When this is true, property values of type array or object generate separate parameters for each value of the array, or key-value-pair of the map. For other types of properties this property has no effect. When style is form, the default value is true. For all other styles, the default value is false. This property SHALL be ignored if the request body media type is not application/x-www-form-urlencoded or multipart/form-data. If a value is explicitly defined, then the value of contentType (implicit or explicit) SHALL be ignored. - allow_reserved bool // Determines whether the parameter value SHOULD allow reserved characters, as defined by [RFC3986] :/?#[]@!$&'()*+,;= to be included without percent-encoding. The default value is false. This property SHALL be ignored if the request body media type is not application/x-www-form-urlencoded or multipart/form-data. If a value is explicitly defined, then the value of contentType (implicit or explicit) SHALL be ignored. -} - -pub struct Parameter { - name string @[required] // The name of the parameter. Parameter names are case sensitive. - in_ string @[json: 'in'; required] // The location of the parameter. Possible values are "query", "header", "path" or "cookie". - description string // A brief description of the parameter. This could contain examples of use. CommonMark syntax MAY be used for rich text representation. - required bool // Determines whether this parameter is mandatory. If the parameter location is "path", this property is REQUIRED and its value MUST be true. Otherwise, the property MAY be included and its default value is false. - deprecated bool // Specifies that a parameter is deprecated and SHOULD be transitioned out of usage. Default value is false. - allow_empty_value bool @[json: 'allowEmptyValue'] // Sets the ability to pass empty-valued parameters. This is valid only for query parameters and allows sending a parameter with an empty value. Default value is false. If style is used, and if behavior is n/a (cannot be serialized), the value of allowEmptyValue SHALL be ignored. Use of this property is NOT RECOMMENDED, as it is likely to be removed in a later revision. - schema Schema // The schema defining the type used for the parameter. -} - -pub struct Example {} - -pub struct SecurityScheme {} - -pub struct RequestBody {} - -pub struct SecurityRequirement {} - -pub struct Tag {} - -pub struct ExternalDocumentation {} diff --git a/crystallib/core/pathlib/path_copy.v b/crystallib/core/pathlib/path_copy.v index 853af554f..1608dbbcf 100644 --- a/crystallib/core/pathlib/path_copy.v +++ b/crystallib/core/pathlib/path_copy.v @@ -1,6 +1,5 @@ module pathlib -import freeflowuniverse.crystallib.ui.console import os @[params] diff --git a/crystallib/core/playbook/parse_action.v b/crystallib/core/playbook/parse_action.v new file mode 100644 index 000000000..7029e3d67 --- /dev/null +++ b/crystallib/core/playbook/parse_action.v @@ -0,0 +1,90 @@ +module playbook + +import freeflowuniverse.crystallib.core.texttools +import freeflowuniverse.crystallib.data.paramsparser + +// TODO: maybe use this in playbook_add? +pub fn parse_single_action(action_text string) !Action { + mut action := Action{} + mut paramsdata := []string{} + mut comments := []string{} + mut state := State.start + + for line_ in action_text.split_into_lines() { + line := line_.replace('\t', ' ') + line_strip := line.trim_space() + + if line_strip.len == 0 { + continue + } + + if state == .action { + if !line.starts_with(' ') || line_strip == '' || line_strip.starts_with('!') { + state = .start + action.params = paramsparser.new(paramsdata.join('\n'))! + action.params.delete('id') + return action + } else { + paramsdata << line + } + } + + if state == .comment_for_action_maybe { + if line.starts_with('//') { + comments << line_strip.trim_left('/ ') + } else { + state = .start + action.comments = comments.join('\n') + comments = []string{} + } + } + + if state == .start { + if line_strip.starts_with('!') && !line_strip.starts_with('![') { + state = .action + action.comments = comments.join('\n') + comments = []string{} + paramsdata = []string{} + mut actionname := line_strip + if line_strip.contains(' ') { + actionname = line_strip.all_before(' ').trim_space() + paramsdata << line_strip.all_after_first(' ').trim_space() + } + if actionname.starts_with('!!!!!') { + return error('There is no action starting with 5 x !') + } else if actionname.starts_with('!!!!') { + action.actiontype = .wal + } else if actionname.starts_with('!!!') { + action.actiontype = .macro + } else if actionname.starts_with('!!') { + action.actiontype = .sal + } else if actionname.starts_with('!') { + action.actiontype = .dal + } else { + return error('Unexpected action type') + } + actionname = actionname.trim_left('!') + splitted := actionname.split('.') + if splitted.len == 1 { + action.actor = 'core' + action.name = texttools.name_fix(splitted[0]) + } else if splitted.len == 2 { + action.actor = texttools.name_fix(splitted[0]) + action.name = texttools.name_fix(splitted[1]) + } else { + return error('Only actions with 1 or 2 parts are supported.\n${actionname}') + } + continue + } else if line.starts_with('//') { + state = .comment_for_action_maybe + comments << line_strip.trim_left('/ ') + } + } + } + // Finalize if still in action state + if state == .action && action.id == 0 { + action.params = paramsparser.new(paramsdata.join('\n'))! + action.params.delete('id') + } + return action +} \ No newline at end of file diff --git a/crystallib/core/playcmds/factory.v b/crystallib/core/playcmds/factory.v index 8dee27431..6462a0130 100644 --- a/crystallib/core/playcmds/factory.v +++ b/crystallib/core/playcmds/factory.v @@ -5,6 +5,7 @@ import freeflowuniverse.crystallib.core.playbook import freeflowuniverse.crystallib.virt.hetzner //import freeflowuniverse.crystallib.clients.b2 import freeflowuniverse.crystallib.biz.bizmodel +import freeflowuniverse.crystallib.hero.publishing import freeflowuniverse.crystallib.threefold.grid4.gridsimulator //import freeflowuniverse.crystallib.installers.sysadmintools.daguserver import freeflowuniverse.crystallib.threefold.grid4.farmingsimulator @@ -35,7 +36,7 @@ pub fn run(mut plbook playbook.PlayBook, dagu bool) ! { // base_install(play(mut plbook)! // coredns.play(mut plbook)! - play_mdbook(mut plbook)! + publishing.play(mut plbook)! //plbook.empty_check()! diff --git a/crystallib/core/playcmds/play_doctree.v b/crystallib/core/playcmds/play_doctree.v new file mode 100644 index 000000000..90ebe8921 --- /dev/null +++ b/crystallib/core/playcmds/play_doctree.v @@ -0,0 +1,70 @@ +module playcmds + +import freeflowuniverse.crystallib.data.doctree +import freeflowuniverse.crystallib.core.playbook +import os + +pub fn play_doctree(mut plbook playbook.PlayBook) ! { + + // check if any actions for doctree, if not then nothing to do here + // dtactions := plbook.find(filter: 'doctree.')! + // if dtactions.len == 0 { + // console.print_debug("can't find doctree.add statements, nothing to do") + // return + // } + + mut trees := map[string]&doctree.Tree{} + for mut action in plbook.find(filter: 'doctree:new')! { + mut p := action.params + name := p.get('name')! + fail_on_error := p.get_default_false('fail_on_error') + println('fail on error: ${fail_on_error}') + if name in trees { + return error('tree with name ${name} already exists') + } + + tree := doctree.new(name: name, fail_on_error: fail_on_error)! + trees[name] = tree + } + + for mut action in plbook.find(filter: 'doctree:add')! { + mut p := action.params + url := p.get_default('url', '')! + path := p.get_default('path', '')! + name := p.get('name')! + + mut tree := trees[name] or { return error('tree ${name} not found') } + + // tree.scan( + // path: path + // git_url: url + // git_reset: reset + // git_root: coderoot + // git_pull: pull + // )! + // action.done = true + } + + for mut action in plbook.find(filter: 'doctree:export')! { + mut p := action.params + build_path := p.get('path')! + toreplace := p.get_default('replace', '')! + reset2 := p.get_default_false('reset') + name := p.get('name')! + mut tree := trees[name] or { return error('tree: ${name} not found') } + + tree.export( + destination: build_path + reset: reset2 + toreplace: toreplace + )! + action.done = true + } + + for mut action in plbook.find(filter: 'doctree:export')! { + panic('implement') + mut p := action.params + name := p.get('name')! + action.done = true + } +} diff --git a/crystallib/core/playcmds/play_mdbook.v b/crystallib/core/playcmds/play_mdbook.v index 57327ba5e..c4a3ada41 100644 --- a/crystallib/core/playcmds/play_mdbook.v +++ b/crystallib/core/playcmds/play_mdbook.v @@ -123,7 +123,6 @@ pub fn play_mdbook(mut plbook playbook.PlayBook) ! { mdbooks.generate( name: name title: title - summary_url: summary_url summary_path: summary_path publish_path: publish_path build_path: build_path diff --git a/crystallib/core/texttools/namefix.v b/crystallib/core/texttools/namefix.v index c21341b39..3c6415047 100644 --- a/crystallib/core/texttools/namefix.v +++ b/crystallib/core/texttools/namefix.v @@ -84,6 +84,11 @@ pub fn name_fix_no_underscore(name string) string { return x } +pub fn name_fix_snake(name string) string { + name_ := name_fix_dot_notation_to_pascal(name) + return name_fix_pascal_to_snake(name_) +} + pub fn name_fix_snake_to_pascal(name string) string { x := name.replace('_', ' ') p := x.title().replace(' ', '') diff --git a/crystallib/data/dbfs/namedb.v b/crystallib/data/dbfs/namedb.v index f3f1d1e8d..9cc314658 100644 --- a/crystallib/data/dbfs/namedb.v +++ b/crystallib/data/dbfs/namedb.v @@ -51,11 +51,11 @@ pub fn (mut db NameDB) save() ! { pub fn (mut db NameDB) set(key string, data string) !u32 { myid, mut mypath := db.key2path(key)! // Check if the pubkey already exists in the file - mut line_num := 0 + mut line_num := u32(0) content := mypath.read()! mut lines := content.trim_space().split_into_lines() mut lines_out := []string{} - mut idfound := 0 + mut idfound := u32(0) for mut line in lines { key_in_file, _ := namedb_process_line(mypath.path, line) if key_in_file == key { @@ -74,7 +74,7 @@ pub fn (mut db NameDB) set(key string, data string) !u32 { } mypath.write(lines.join('\n'))! - return u32(myid + lines.len - 1) + return myid + u32(lines.len) - 1 } pub fn (mut db NameDB) delete(key string) ! { @@ -99,13 +99,13 @@ pub fn (mut db NameDB) delete(key string) ! { // will store in a place where it can easily be found back and it returns a unique u32 pub fn (mut db NameDB) get(key string) !(u32, string) { myid, mut mypath := db.key2path(key)! - mut line_num := 0 + mut line_num := u32(0) content := mypath.read()! mut lines := content.trim_space().split_into_lines() for line in lines { key_in_file, data := namedb_process_line(mypath.path, line) if key_in_file == key { - return u32(myid + line_num), data + return myid + line_num, data } line_num += 1 } @@ -146,10 +146,10 @@ pub fn (mut db NameDB) get_from_id(myid u32) !(string, string) { // calculate the id's as needed to create the path fn namedb_dbid(myid u32) (u8, u8, u16) { - a := u8(myid / (256 * 256)) - a_post := myid - a * int(256 * 256) + a := u8(myid / u32(256 * 256)) + a_post := myid - u32(a) * u32(256 * 256) b := u8(a_post / 256) - b_post := a_post - b * int(256) + b_post := a_post - u32(b) * u32(256) c := u16(b_post) return a, b, c } @@ -161,9 +161,9 @@ fn (mut db NameDB) key2path(key string) !(u32, pathlib.Path) { } a := hash_bytes[0] or { panic('bug') } b := hash_bytes[1] or { panic('bug') } - mut myid := int(a) * 256 * 256 + int(b) * 256 - mut mypath := db.dbpath(u32(myid))! - return u32(myid), mypath + myid := u32(int(a) * 256 * 256 + int(b) * 256) + mut mypath := db.dbpath(myid)! + return myid, mypath } fn namedb_process_line(path string, line string) (string, string) { diff --git a/crystallib/data/dbfs/readme.md b/crystallib/data/dbfs/readme.md index 33f2d5914..b8fa567d0 100644 --- a/crystallib/data/dbfs/readme.md +++ b/crystallib/data/dbfs/readme.md @@ -13,7 +13,7 @@ The algo's used have been optimized for scalability and human readability, the i > TODO: fix, we refactored -import freeflowuniverse.crystallib.core.dbfs +import freeflowuniverse.crystallib.data.dbfs mut dbcollection := get(context: 'test', secret: '123456')! @@ -42,7 +42,7 @@ e.g. ideal for config sessions (which are done on context level) > TODO: fix, we refactored -import freeflowuniverse.crystallib.core.dbfs +import freeflowuniverse.crystallib.data.dbfs mut dbcollection := get(context: 'test', secret: '123456')! diff --git a/crystallib/data/doctree/README.md b/crystallib/data/doctree/README.md new file mode 100644 index 000000000..84f320323 --- /dev/null +++ b/crystallib/data/doctree/README.md @@ -0,0 +1,2 @@ +# doctree + diff --git a/crystallib/data/doctree/collection/collection.v b/crystallib/data/doctree/collection/collection.v index 9202b60e7..f88fdb3a6 100644 --- a/crystallib/data/doctree/collection/collection.v +++ b/crystallib/data/doctree/collection/collection.v @@ -7,8 +7,8 @@ import freeflowuniverse.crystallib.core.texttools @[heap] pub struct Collection { pub mut: - name string @[required] - path Path @[required] + name string @[required] + path Path @[required] fail_on_error bool heal bool = true pages map[string]&data.Page @@ -34,15 +34,26 @@ pub fn new(args_ CollectionNewArgs) !Collection { mut pp := pathlib.get_dir(path: args.path)! // will raise error if path doesn't exist mut collection := Collection{ - name: args.name - path: pp - heal: args.heal + name: args.name + path: pp + heal: args.heal fail_on_error: args.fail_on_error } if args.load { - collection.scan()! + collection.scan() or { return error('Error scanning collection ${args.name}:\n${err}') } } return collection } + +fn (c Collection) get_linked_pages() ![]string { + mut linked_pages_set := map[string]bool{} + for _, page in c.pages { + for linked_page in page.get_linked_pages()! { + linked_pages_set[linked_page] = true + } + } + + return linked_pages_set.keys() +} diff --git a/crystallib/data/doctree/collection/data/file.v b/crystallib/data/doctree/collection/data/file.v index b9c2e1513..ef0209bcd 100644 --- a/crystallib/data/doctree/collection/data/file.v +++ b/crystallib/data/doctree/collection/data/file.v @@ -35,15 +35,15 @@ pub: collection_path pathlib.Path pathrel string path pathlib.Path - collection_name string @[required] + collection_name string @[required] } pub fn new_file(args NewFileArgs) !File { mut f := File{ - name: args.name - path: args.path + name: args.name + path: args.path collection_path: args.collection_path - pathrel: args.pathrel + pathrel: args.pathrel collection_name: args.collection_name } @@ -93,9 +93,22 @@ fn (mut file File) exists() !bool { return file.path.exists() } -pub fn (mut file File) copy(dest string) ! { +pub fn (file_ File) copy(dest string) ! { + mut file := file_ mut dest2 := pathlib.get(dest) file.path.copy(dest: dest2.path, rsync: false) or { return error('Could not copy file: ${file.path.path} to ${dest} .\n${err}\n${file}') } } + +pub struct ExportParams { +pub: + reset bool // whether the export will overwrite +} + +pub fn (file File) export(dest string, params ExportParams) ! { + d := '${dest}/${file.name}.${file.ext}' + if params.reset || !os.exists(d) { + file.copy(d)! + } +} diff --git a/crystallib/data/doctree/collection/data/page.v b/crystallib/data/doctree/collection/data/page.v index ec3a8a62c..25caa70dd 100644 --- a/crystallib/data/doctree/collection/data/page.v +++ b/crystallib/data/doctree/collection/data/page.v @@ -3,6 +3,7 @@ module data import freeflowuniverse.crystallib.core.pathlib import freeflowuniverse.crystallib.data.markdownparser.elements { Action, Doc, Element } import freeflowuniverse.crystallib.data.markdownparser +import freeflowuniverse.crystallib.core.texttools.regext pub enum PageStatus { unknown @@ -13,7 +14,7 @@ pub enum PageStatus { @[heap] pub struct Page { mut: - doc &Doc @[str: skip] + doc &Doc @[str: skip] element_cache map[int]Element changed bool pub mut: @@ -39,18 +40,20 @@ pub fn new_page(args NewPageArgs) !Page { if args.name == '' { return error('page name must not be empty') } - mut doc := markdownparser.new(path: args.path.path, collection_name: args.collection_name)! + mut doc := markdownparser.new(path: args.path.path, collection_name: args.collection_name) or { + return error('failed to parse doc for path ${args.path.path}\n${err}') + } children := doc.children_recursive() mut element_cache := map[int]Element{} for child in children { element_cache[child.id] = child } mut new_page := Page{ - element_cache: element_cache - name: args.name - path: args.path + element_cache: element_cache + name: args.name + path: args.path collection_name: args.collection_name - doc: &doc + doc: &doc } return new_page } @@ -65,6 +68,16 @@ fn (mut page Page) doc() !&Doc { return page.doc } +// return doc, reparse if needed +fn (page Page) doc_immute() !&Doc { + if page.changed { + content := page.doc.markdown()! + doc := markdownparser.new(content: content, collection_name: page.collection_name)! + return &doc + } + return page.doc +} + // reparse doc markdown and assign new doc to page fn (mut page Page) reparse_doc(content string) ! { doc := markdownparser.new(content: content, collection_name: page.collection_name)! @@ -81,13 +94,13 @@ pub fn (page Page) key() string { return '${page.collection_name}:${page.name}' } -pub fn (mut page Page) get_linked_pages() ![]string { - doc := page.doc()! +pub fn (page Page) get_linked_pages() ![]string { + doc := page.doc_immute()! return doc.linked_pages } -pub fn (mut page Page) get_markdown() !string { - mut doc := page.doc()! +pub fn (page Page) get_markdown() !string { + mut doc := page.doc_immute()! return doc.markdown()! } @@ -114,17 +127,17 @@ pub fn (mut page Page) get_all_actions() ![]&Action { return actions } -pub fn (mut page Page) get_include_actions() ![]Action { +pub fn (page Page) get_include_actions() ![]Action { mut actions := []Action{} - mut doc := page.doc()! - for element in doc.children_recursive() { + // TODO: check if below is necessary + mut doc := page.doc_immute()! + for element in page.doc.children_recursive() { if element is Action { if element.action.actor == 'wiki' && element.action.name == 'include' { actions << *element } } } - return actions } @@ -150,3 +163,25 @@ pub fn (mut page Page) set_element_content_no_reparse(element_id int, content st element.content = content page.changed = true } + +@[params] +pub struct ExportPageParams { +pub mut: + dir_src pathlib.Path + file_paths map[string]string + keep_structure bool // wether the structure of the src collection will be preserved or not + replacer ?regext.ReplaceInstructions +} + +pub fn (p Page) export(directory string, params ExportPageParams) ! { + // TODO: implement export with keep structure, maybe higher + dest := '${directory}/${p.name}.md' + + mut dest_path := pathlib.get_file(path: dest, create: true)! + mut markdown := p.get_markdown()! + if mut replacer := params.replacer { + markdown = replacer.replace(text: markdown)! + } + + dest_path.write(markdown)! +} diff --git a/crystallib/data/doctree/collection/data/process_link.v b/crystallib/data/doctree/collection/data/process_link.v index eefcd46b3..10b20f401 100644 --- a/crystallib/data/doctree/collection/data/process_link.v +++ b/crystallib/data/doctree/collection/data/process_link.v @@ -5,11 +5,16 @@ import freeflowuniverse.crystallib.data.markdownparser.elements import freeflowuniverse.crystallib.data.doctree.pointer // Note: doc should not get reparsed after invoking this method -pub fn (mut page Page) process_links(paths map[string]string) ![]string { +pub fn (page Page) process_links(paths map[string]string) ![]string { mut not_found := map[string]bool{} - mut doc := page.doc()! + mut doc := page.doc_immute()! for mut element in doc.children_recursive() { if mut element is elements.Link { + if element.cat == .html || (element.cat == .anchor && element.url == '') { + // is external link or same page anchor, nothing to process + // maybe in the future check if exists + continue + } mut name := texttools.name_fix_keepext(element.filename) mut site := texttools.name_fix(element.site) if site == '' { diff --git a/crystallib/data/doctree/collection/error.v b/crystallib/data/doctree/collection/error.v index d927386df..dc27b18ac 100644 --- a/crystallib/data/doctree/collection/error.v +++ b/crystallib/data/doctree/collection/error.v @@ -1,6 +1,7 @@ module collection import freeflowuniverse.crystallib.core.pathlib { Path } +import freeflowuniverse.crystallib.data.doctree.pointer { Pointer } import freeflowuniverse.crystallib.ui.console pub enum CollectionErrorCat { @@ -52,13 +53,32 @@ pub fn (err ObjNotFound) msg() string { } // write errors.md in the collection, this allows us to see what the errors are -pub fn (collection Collection) errors_report(dest_ string) ! { +pub fn (collection Collection) errors_report(dest_ string, errors []CollectionError) ! { // console.print_debug("====== errors report: ${dest_} : ${collection.errors.len}\n${collection.errors}") mut dest := pathlib.get_file(path: dest_, create: true)! - if collection.errors.len == 0 { + if errors.len == 0 { dest.delete()! return } c := $tmpl('template/errors.md') dest.write(c)! } + +fn error_pointer_not_found(ptr Pointer) CollectionError { + cat := match ptr.cat { + .page { + CollectionErrorCat.page_not_found + } + .image { + CollectionErrorCat.image_not_found + } + else { + CollectionErrorCat.file_not_found + } + } + + return CollectionError{ + msg: '${ptr.cat} ${ptr.str()} not found' + cat: cat + } +} diff --git a/crystallib/data/doctree/collection/export.v b/crystallib/data/doctree/collection/export.v index 50c93d5f1..e6b44e433 100644 --- a/crystallib/data/doctree/collection/export.v +++ b/crystallib/data/doctree/collection/export.v @@ -4,11 +4,12 @@ import freeflowuniverse.crystallib.core.pathlib import freeflowuniverse.crystallib.core.texttools.regext import os import freeflowuniverse.crystallib.data.doctree.pointer +import freeflowuniverse.crystallib.data.doctree.collection.data @[params] pub struct CollectionExportArgs { pub mut: - destination pathlib.Path @[required] + destination pathlib.Path @[required] file_paths map[string]string reset bool = true keep_structure bool // wether the structure of the src collection will be preserved or not @@ -16,104 +17,41 @@ pub mut: replacer ?regext.ReplaceInstructions } -pub fn (mut c Collection) export(args CollectionExportArgs) ! { +pub fn (c Collection) export(args CollectionExportArgs) ! { dir_src := pathlib.get_dir(path: args.destination.path + '/' + c.name, create: true)! mut cfile := pathlib.get_file(path: dir_src.path + '/.collection', create: true)! // will auto save it cfile.write("name:${c.name} src:'${c.path.path}'")! - c.export_pages( - dir_src: dir_src - file_paths: args.file_paths - keep_structure: args.keep_structure - replacer: args.replacer - )! - c.export_files(dir_src, args.reset)! - c.export_images(dir_src, args.reset)! - c.export_linked_pages(dir_src)! + mut errors := c.errors.clone() - if !args.exclude_errors { - c.errors_report('${dir_src.path}/errors.md')! - } -} - -@[params] -pub struct ExportPagesArgs { -pub mut: - dir_src pathlib.Path - file_paths map[string]string - keep_structure bool // wether the structure of the src collection will be preserved or not - replacer ?regext.ReplaceInstructions -} - -// creates page file, processes page links, then writes page -fn (mut c Collection) export_pages(args ExportPagesArgs) ! { - for _, mut page in c.pages { - dest := if args.keep_structure { - relpath := page.path.path.trim_string_left(c.path.path) - '${args.dir_src.path}/${relpath}' - } else { - '${args.dir_src.path}/${page.name}.md' - } - - not_found := page.process_links(args.file_paths)! - for pointer_str in not_found { - ptr := pointer.pointer_new(text: pointer_str)! - cat := match ptr.cat { - .page { - CollectionErrorCat.page_not_found - } - .image { - CollectionErrorCat.image_not_found - } - else { - CollectionErrorCat.file_not_found - } + for _, page in c.pages { + page.export(dir_src.path, + file_paths: args.file_paths + keep_structure: args.keep_structure + replacer: args.replacer + ) or { + if err is CollectionError { + errors << err } - c.error(path: page.path, msg: '${ptr.cat} ${ptr.str()} not found', cat: cat)! } - - mut dest_path := pathlib.get_file(path: dest, create: true)! - mut markdown := page.get_markdown()! - if mut replacer := args.replacer { - markdown = replacer.replace(text: markdown)! - } - - dest_path.write(markdown)! } -} -fn (mut c Collection) export_files(dir_src pathlib.Path, reset bool) ! { - for _, mut file in c.files { - mut d := '${dir_src.path}/img/${file.name}.${file.ext}' - if reset || !os.exists(d) { - file.copy(d)! - } + // export files and images + for _, file in c.files { + file.export('${dir_src.path}/file/', reset: args.reset)! } -} - -fn (mut c Collection) export_images(dir_src pathlib.Path, reset bool) ! { - for _, mut file in c.images { - mut d := '${dir_src.path}/img/${file.name}.${file.ext}' - if reset || !os.exists(d) { - file.copy(d)! - } + for _, image in c.images { + image.export('${dir_src.path}/img/', reset: args.reset)! } -} -fn (mut c Collection) export_linked_pages(dir_src pathlib.Path) ! { - collection_linked_pages := c.get_collection_linked_pages()! - mut linked_pages_file := pathlib.get_file(path: dir_src.path + '/.linkedpages', create: true)! - linked_pages_file.write(collection_linked_pages.join_lines())! -} + // export the metadata of pages linked in collection + linked_pages := c.get_linked_pages()! + mut linked_pages_file := pathlib.get_file(path: '${dir_src.path}/.linkedpages', create: true)! + linked_pages_file.write(linked_pages.join_lines())! -fn (mut c Collection) get_collection_linked_pages() ![]string { - mut linked_pages_set := map[string]bool{} - for _, mut page in c.pages { - for linked_page in page.get_linked_pages()! { - linked_pages_set[linked_page] = true - } + if !args.exclude_errors { + c.errors_report('${dir_src.path}/errors.md', errors)! } - - return linked_pages_set.keys() } + diff --git a/crystallib/data/doctree/collection/export_test.v b/crystallib/data/doctree/collection/export_test.v index 6f671badb..a4275038a 100644 --- a/crystallib/data/doctree/collection/export_test.v +++ b/crystallib/data/doctree/collection/export_test.v @@ -10,14 +10,14 @@ const export_expected_dir = '${test_dir}/export_expected' fn testsuite_begin() { pathlib.get_dir( - path: export_dir + path: export_dir empty: true )! } fn testsuite_end() { pathlib.get_dir( - path: export_dir + path: export_dir empty: true )! } @@ -32,7 +32,7 @@ fn test_export() { path_dest := pathlib.get_dir(path: '${export_dir}/src', create: true)! col.export( destination: path_dest - file_paths: { + file_paths: { 'col2:file3.md': 'col2/file3.md' } )! diff --git a/crystallib/data/doctree/collection/getters.v b/crystallib/data/doctree/collection/getters.v index de6a82100..9716c0c0b 100644 --- a/crystallib/data/doctree/collection/getters.v +++ b/crystallib/data/doctree/collection/getters.v @@ -7,7 +7,7 @@ pub fn (collection Collection) page_get(name string) !&data.Page { return collection.pages[name] or { return ObjNotFound{ collection: collection.name - name: name + name: name } } } @@ -21,7 +21,7 @@ pub fn (collection Collection) get_image(name string) !&data.File { return collection.images[name] or { return ObjNotFound{ collection: collection.name - name: name + name: name } } } @@ -35,7 +35,7 @@ pub fn (collection Collection) get_file(name string) !&data.File { return collection.files[name] or { return ObjNotFound{ collection: collection.name - name: name + name: name } } } diff --git a/crystallib/data/doctree/collection/scan.v b/crystallib/data/doctree/collection/scan.v index ac441fa95..6656d9acb 100644 --- a/crystallib/data/doctree/collection/scan.v +++ b/crystallib/data/doctree/collection/scan.v @@ -21,8 +21,8 @@ fn (mut collection Collection) scan_directory(mut p Path) ! { if !entry.exists() { collection.error( path: entry - msg: 'Entry ${entry.name()} does not exists' - cat: .unknown + msg: 'Entry ${entry.name()} does not exists' + cat: .unknown )! continue } @@ -34,9 +34,9 @@ fn (mut collection Collection) scan_directory(mut p Path) ! { // means we are linking pages,this should not be done, need or change collection.error( path: entry - msg: 'Markdown files (${entry.path}) must not be linked' - cat: .unknown - )! + msg: 'Markdown files (${entry.path}) must not be linked' + cat: .unknown + ) or { return error('Failed to collection error ${entry.path}:\n${err}') } continue } @@ -50,7 +50,9 @@ fn (mut collection Collection) scan_directory(mut p Path) ! { } if entry.is_dir() { - collection.scan_directory(mut entry)! + collection.scan_directory(mut entry) or { + return error('Failed to scan directory ${entry.path}:\n${err}') + } continue } @@ -60,10 +62,14 @@ fn (mut collection Collection) scan_directory(mut p Path) ! { match entry.extension_lower() { 'md' { - collection.add_page(mut entry)! + collection.add_page(mut entry) or { + return error('Failed to add page ${entry.path}:\n${err}') + } } else { - collection.file_image_remember(mut entry)! + collection.file_image_remember(mut entry) or { + return error('Failed to remember image ${entry.path}:\n${err}') + } } } } @@ -111,7 +117,7 @@ fn (mut collection Collection) file_image_remember(mut p Path) ! { } mut ptr := pointer.pointer_new( collection: collection.name - text: p.name() + text: p.name() )! if ptr.is_file_video_html() { @@ -160,29 +166,29 @@ fn (mut collection Collection) file_image_remember(mut p Path) ! { // the page will be parsed as markdown pub fn (mut collection Collection) add_page(mut p Path) ! { if collection.heal { - p.path_normalize()! + p.path_normalize() or { return error('Failed to normalize path ${p.path}\n${err}') } } mut ptr := pointer.pointer_new( collection: collection.name - text: p.name() - )! + text: p.name() + ) or { return error('Failed to get pointer for ${p.name()}\n${err}') } // in case heal is true pointer_new can normalize the path if collection.page_exists(ptr.name) { collection.error( path: p - msg: 'Can\'t add ${p.path}: a page named ${ptr.name} already exists in the collection' - cat: .page_double - )! + msg: 'Can\'t add ${p.path}: a page named ${ptr.name} already exists in the collection' + cat: .page_double + ) or { return error('Failed to report collection error for ${p.name()}\n${err}') } return } new_page := data.new_page( - name: ptr.name - path: p + name: ptr.name + path: p collection_name: collection.name - )! + ) or { return error('Failed to create new page for ${ptr.name}\n${err}') } collection.pages[ptr.name] = &new_page } @@ -194,21 +200,21 @@ pub fn (mut collection Collection) add_file(mut p Path) ! { } mut ptr := pointer.pointer_new( collection: collection.name - text: p.name() + text: p.name() )! // in case heal is true pointer_new can normalize the path if collection.file_exists(ptr.name) { collection.error( path: p - msg: 'Can\'t add ${p.path}: a file named ${ptr.name} already exists in the collection' - cat: .file_double + msg: 'Can\'t add ${p.path}: a file named ${ptr.name} already exists in the collection' + cat: .file_double )! return } mut new_file := data.new_file( - path: p + path: p collection_path: collection.path collection_name: collection.name )! @@ -222,21 +228,21 @@ pub fn (mut collection Collection) add_image(mut p Path) ! { } mut ptr := pointer.pointer_new( collection: collection.name - text: p.name() + text: p.name() )! // in case heal is true pointer_new can normalize the path if collection.image_exists(ptr.name) { collection.error( path: p - msg: 'Can\'t add ${p.path}: a file named ${ptr.name} already exists in the collection' - cat: .image_double + msg: 'Can\'t add ${p.path}: a file named ${ptr.name} already exists in the collection' + cat: .image_double )! return } mut image_file := &data.File{ - path: p + path: p collection_path: collection.path } image_file.init()! diff --git a/crystallib/data/doctree/export.v b/crystallib/data/doctree/export.v index 9e5b12750..81dd66ee7 100644 --- a/crystallib/data/doctree/export.v +++ b/crystallib/data/doctree/export.v @@ -1,6 +1,8 @@ module doctree import freeflowuniverse.crystallib.core.pathlib +import freeflowuniverse.crystallib.data.doctree.collection { Collection } +import freeflowuniverse.crystallib.data.doctree.collection.data import freeflowuniverse.crystallib.ui.console import freeflowuniverse.crystallib.core.texttools.regext @@ -12,6 +14,7 @@ pub mut: keep_structure bool // wether the structure of the src collection will be preserved or not exclude_errors bool // wether error reporting should be exported as well toreplace string + concurrent bool = true } // export all collections to chosen directory . @@ -33,19 +36,37 @@ pub fn (mut tree Tree) export(args TreeExportArgs) ! { tree.process_defs()! tree.process_includes()! tree.process_actions_and_macros()! // process other actions and macros - - file_paths := tree.generate_paths()! + tree.process_links()! console.print_green('exporting collections') - for _, mut collection in tree.collections { - collection.export( - destination: dest_path - file_paths: file_paths - reset: args.reset - keep_structure: args.keep_structure - exclude_errors: args.exclude_errors - replacer: tree.replacer - )! + + if args.concurrent { + mut ths := []thread !{} + for _, col in tree.collections { + ths << spawn fn (col Collection, dest_path pathlib.Path, file_paths map[string]string, args TreeExportArgs) ! { + col.export( + destination: dest_path + reset: args.reset + keep_structure: args.keep_structure + exclude_errors: args.exclude_errors + // TODO: replacer: tree.replacer + )! + }(col, dest_path, file_paths, args) + } + for th in ths { + th.wait() or { panic(err) } + } + } else { + for _, mut col in tree.collections { + col.export( + destination: dest_path + file_paths: file_paths + reset: args.reset + keep_structure: args.keep_structure + exclude_errors: args.exclude_errors + replacer: tree.replacer + )! + } } } diff --git a/crystallib/data/doctree/export_test.v b/crystallib/data/doctree/export_test.v index 346d29b3e..c8be69091 100644 --- a/crystallib/data/doctree/export_test.v +++ b/crystallib/data/doctree/export_test.v @@ -10,14 +10,14 @@ const export_expected_dir = '${test_dir}/export_expected' fn testsuite_begin() { pathlib.get_dir( - path: export_dir + path: export_dir empty: true )! } fn testsuite_end() { pathlib.get_dir( - path: export_dir + path: export_dir empty: true )! } diff --git a/crystallib/data/doctree/getters.v b/crystallib/data/doctree/getters.v index 2c35239fd..18cdc2032 100644 --- a/crystallib/data/doctree/getters.v +++ b/crystallib/data/doctree/getters.v @@ -14,7 +14,7 @@ pub fn (tree Tree) get_collection_with_pointer(p pointer.Pointer) !&collection.C return tree.get_collection(p.collection) or { return CollectionNotFound{ pointer: p - msg: '${err}' + msg: '${err}' } } } diff --git a/crystallib/data/doctree/pointer/pointer.v b/crystallib/data/doctree/pointer/pointer.v index 5a173e7e1..13ffdadf6 100644 --- a/crystallib/data/doctree/pointer/pointer.v +++ b/crystallib/data/doctree/pointer/pointer.v @@ -86,10 +86,10 @@ pub fn pointer_new(args NewPointerArgs) !Pointer { } return Pointer{ - name: file_name_no_extension + name: file_name_no_extension collection: collection_name - extension: extension - cat: file_cat + extension: extension + cat: file_cat } } diff --git a/crystallib/data/doctree/process_defs.v b/crystallib/data/doctree/process_defs.v index 07038acab..f78257ad1 100644 --- a/crystallib/data/doctree/process_defs.v +++ b/crystallib/data/doctree/process_defs.v @@ -1,6 +1,6 @@ module doctree -import freeflowuniverse.crystallib.data.doctree.collection {CollectionError} +import freeflowuniverse.crystallib.data.doctree.collection { CollectionError } import freeflowuniverse.crystallib.data.doctree.collection.data import freeflowuniverse.crystallib.ui.console @@ -34,8 +34,8 @@ fn (mut tree Tree) process_page_def_actions(mut p data.Page, mut c collection.Co if def_actions.len > 1 { c.error( path: p.path - msg: 'a page can have at most one def action' - cat: .def + msg: 'a page can have at most one def action' + cat: .def )! } @@ -48,8 +48,8 @@ fn (mut tree Tree) process_page_def_actions(mut p data.Page, mut c collection.Co if alias in tree.defs { c.error( path: p.path - msg: 'alias ${alias} is already used' - cat: .def + msg: 'alias ${alias} is already used' + cat: .def )! continue } @@ -68,7 +68,11 @@ fn (mut tree Tree) replace_page_defs_with_links(mut p data.Page) ![]CollectionEr def_data[def] = [referenced_page.key(), referenced_page.alias] } else { // accrue errors that occur - errors << CollectionError{path: p.path, msg: 'def ${def} is not defined', cat: .def} + errors << CollectionError{ + path: p.path + msg: 'def ${def} is not defined' + cat: .def + } continue } } @@ -76,4 +80,4 @@ fn (mut tree Tree) replace_page_defs_with_links(mut p data.Page) ![]CollectionEr p.set_def_links(def_data)! // return accrued collection errors for collection to handle return errors -} \ No newline at end of file +} diff --git a/crystallib/data/doctree/process_includes.v b/crystallib/data/doctree/process_includes.v index e6fcf4cb3..9e6aef1be 100644 --- a/crystallib/data/doctree/process_includes.v +++ b/crystallib/data/doctree/process_includes.v @@ -2,6 +2,8 @@ module doctree // import freeflowuniverse.crystallib.data.doctree.collection.data import freeflowuniverse.crystallib.data.doctree.pointer +import freeflowuniverse.crystallib.data.doctree.collection { CollectionError } +import freeflowuniverse.crystallib.data.doctree.collection.data import freeflowuniverse.crystallib.core.playbook import freeflowuniverse.crystallib.ui.console @@ -38,7 +40,7 @@ pub fn (mut tree Tree) process_includes() ! { // process page for element in page.get_include_actions()! { - page_pointer := tree.get_include_page_pointer(col.name, element.action) or { continue } + page_pointer := get_include_page_pointer(col.name, element.action) or { continue } mut include_page := tree.get_page_with_pointer(page_pointer) or { continue } @@ -63,13 +65,13 @@ pub fn (mut tree Tree) process_includes() ! { mut col := tree.get_collection(page.collection_name)! col.error( path: page.path - msg: 'page ${key} is in an include cycle' - cat: .circular_import + msg: 'page ${key} is in an include cycle' + cat: .circular_import )! } } -fn (mut tree Tree) get_include_page_pointer(collection_name string, a playbook.Action) !pointer.Pointer { +fn get_include_page_pointer(collection_name string, a playbook.Action) !pointer.Pointer { mut page_pointer_str := a.params.get('page')! // handle includes @@ -83,36 +85,69 @@ fn (mut tree Tree) get_include_page_pointer(collection_name string, a playbook.A fn (mut tree Tree) generate_pages_graph() !map[string]map[string]bool { mut graph := map[string]map[string]bool{} - for _, mut collection in tree.collections { - for _, mut page in collection.pages { - mut current_page := page - _ := graph[current_page.key()] or { - map[string]bool{} - } - include_action_elements := current_page.get_include_actions()! - for element in include_action_elements { - page_pointer := tree.get_include_page_pointer(collection.name, element.action) or { - collection.error( - path: current_page.path - msg: 'failed to get page pointer for include ${element.action.heroscript()}: ${err}' - cat: .include - )! - continue - } - - include_page := tree.get_page_with_pointer(page_pointer) or { - collection.error( - path: current_page.path - msg: 'failed to get page for include ${element.action.heroscript()}: ${err.msg()}' - cat: .include - )! - continue - } - - graph[include_page.key()][current_page.key()] = true - } + mut ths := []thread !map[string]map[string]bool{} + for _, mut col in tree.collections { + ths << spawn fn (mut tree Tree, col &collection.Collection) !map[string]map[string]bool { + return tree.collection_page_graph(col)! + }(mut tree, col) + } + for th in ths { + col_graph := th.wait()! + for k, v in col_graph { + graph[k] = v.clone() + } + } + return graph +} + +fn (mut tree Tree) collection_page_graph(col &collection.Collection) !map[string]map[string]bool { + mut graph := map[string]map[string]bool{} + mut ths := []thread !GraphResponse{} + for _, page in col.pages { + resp := tree.generate_page_graph(page, col.name)! + for k, v in resp.graph { + graph[k] = v.clone() } } return graph } + +pub struct GraphResponse { +pub: + graph map[string]map[string]bool + errors []CollectionError +} + +fn (tree Tree) generate_page_graph(current_page &data.Page, col_name string) !GraphResponse { + mut graph := map[string]map[string]bool{} + mut errors := []CollectionError{} + + include_action_elements := current_page.get_include_actions()! + for element in include_action_elements { + page_pointer := get_include_page_pointer(col_name, element.action) or { + errors << CollectionError{ + path: current_page.path + msg: 'failed to get page pointer for include ${element.action.heroscript()}: ${err}' + cat: .include + } + continue + } + + include_page := tree.get_page_with_pointer(page_pointer) or { + // TODO + // col.error( + // path: current_page.path + // msg: 'failed to get page for include ${element.action.heroscript()}: ${err.msg()}' + // cat: .include + // )! + continue + } + + graph[include_page.key()][current_page.key()] = true + } + return GraphResponse{ + graph: graph + errors: errors + } +} diff --git a/crystallib/data/doctree/process_links.v b/crystallib/data/doctree/process_links.v new file mode 100644 index 000000000..a349c9662 --- /dev/null +++ b/crystallib/data/doctree/process_links.v @@ -0,0 +1,15 @@ +module doctree + +pub fn (mut tree Tree) process_links() ! { + file_paths := tree.generate_paths()! + for _, mut c in tree.collections { + for _, p in c.pages { + not_found := p.process_links(file_paths)! + for pointer_str in not_found { + ptr := pointer.pointer_new(text: pointer_str)! + c.error(error_pointer_not_found(ptr)) + } + } + } + +} \ No newline at end of file diff --git a/crystallib/data/doctree/process_macros.v b/crystallib/data/doctree/process_macros.v index b372f18c4..164c2d457 100644 --- a/crystallib/data/doctree/process_macros.v +++ b/crystallib/data/doctree/process_macros.v @@ -1,5 +1,6 @@ module doctree +import freeflowuniverse.crystallib.data.doctree.collection { Collection } import freeflowuniverse.crystallib.data.markdownparser.elements import freeflowuniverse.crystallib.ui.console import freeflowuniverse.crystallib.core.playbook @@ -26,10 +27,17 @@ pub fn (mut tree Tree) process_actions_and_macros() ! { playmacros.play_actions(mut plbook)! // now get specific actions which need to return content - for _, mut collection in tree.collections { - for _, mut page in collection.pages { - page.process_macros()! // calls play_macro in playmacros... - } + mut ths := []thread !{} + for _, mut col in tree.collections { + ths << spawn fn (mut col Collection) ! { + for _, mut page in col.pages { + page.process_macros()! // calls play_macro in playmacros... + } + }(mut col) + } + + for th in ths { + th.wait()! } } diff --git a/crystallib/data/doctree/scan.v b/crystallib/data/doctree/scan.v index d9cecc0ee..33a5b2a38 100644 --- a/crystallib/data/doctree/scan.v +++ b/crystallib/data/doctree/scan.v @@ -1,8 +1,8 @@ module doctree -import freeflowuniverse.crystallib.core.pathlib +import freeflowuniverse.crystallib.core.pathlib { Path } import freeflowuniverse.crystallib.data.paramsparser -import freeflowuniverse.crystallib.data.doctree.collection +import freeflowuniverse.crystallib.data.doctree.collection { Collection, CollectionNewArgs } import freeflowuniverse.crystallib.develop.gittools import os import freeflowuniverse.crystallib.core.texttools @@ -33,13 +33,13 @@ pub fn (mut tree Tree) scan(args_ TreeScannerArgs) ! { mut args := args_ if args.git_url.len > 0 { mut gs := gittools.get(coderoot: args.git_root)! - mut repo := gs.get_repo( - url: args.git_url - pull: args.git_pull - reset: args.git_reset + mut repo := gs.get_repo( + url: args.git_url + pull: args.git_pull + reset: args.git_reset reload: false )! - args.path = repo.get_path()! + args.path = repo.get_path_of_url(args.git_url)! } if args.path.len == 0 { @@ -52,17 +52,17 @@ pub fn (mut tree Tree) scan(args_ TreeScannerArgs) ! { } if path.file_exists('.site') { - tree.move_site_to_collection(mut path)! + move_site_to_collection(mut path)! } - if tree.is_collection_dir(path) { - collection_name := tree.get_collection_name(mut path)! + if is_collection_dir(path) { + collection_name := get_collection_name(mut path)! tree.add_collection( - path: path.path - name: collection_name - heal: args.heal - load: true + path: path.path + name: collection_name + heal: args.heal + load: true fail_on_error: tree.fail_on_error )! @@ -74,7 +74,7 @@ pub fn (mut tree Tree) scan(args_ TreeScannerArgs) ! { } for mut entry in entries.paths { - if !entry.is_dir() || tree.is_ignored_dir(mut entry)! { + if !entry.is_dir() || is_ignored_dir(entry)! { continue } @@ -84,14 +84,83 @@ pub fn (mut tree Tree) scan(args_ TreeScannerArgs) ! { } } -@[params] -pub struct CollectionNewArgs { -mut: - name string @[required] - path string @[required] - heal bool = true // healing means we fix images, if selected will automatically load, remove stale links - load bool = true - fail_on_error bool +pub fn (mut tree Tree) scan_concurrent(args_ TreeScannerArgs) ! { + mut args := args_ + if args.git_url.len > 0 { + mut gs := gittools.get(coderoot: args.git_root)! + mut repo := gs.get_repo( + url: args.git_url + pull: args.git_pull + reset: args.git_reset + reload: false + )! + args.path = repo.get_path_of_url(args.git_url)! + } + + if args.path.len == 0 { + return error('Path needs to be provided.') + } + + path := pathlib.get_dir(path: args.path)! + mut collection_paths := scan_helper(path)! + mut threads := []thread !Collection{} + for mut col_path in collection_paths { + mut col_name := get_collection_name(mut col_path)! + col_name = texttools.name_fix(col_name) + + if col_name in tree.collections { + if tree.fail_on_error { + return error('Collection with name ${col_name} already exits') + } + // TODO: handle error + continue + } + + threads << spawn fn (args CollectionNewArgs) !Collection { + return collection.new(args)! + }( + name: col_name + path: col_path.path + heal: args.heal + fail_on_error: tree.fail_on_error + ) + } + + for i, t in threads { + new_collection := t.wait() or { return error('Error executing thread: ${err}') } + tree.collections[new_collection.name] = &new_collection + } +} + +// internal function that recursively returns +// the paths of collections in a given path +fn scan_helper(path_ Path) ![]Path { + mut path := path_ + if !path.is_dir() { + return error('path is not a directory') + } + + if path.file_exists('.site') { + move_site_to_collection(mut path)! + } + + if is_collection_dir(path) { + return [path] + } + + mut entries := path.list(recursive: false) or { + return error('cannot list: ${path.path} \n${error}') + } + + mut paths := []Path{} + for mut entry in entries.paths { + if !entry.is_dir() || is_ignored_dir(entry)! { + continue + } + + paths << scan_helper(entry) or { return error('failed to scan ${entry.path} :${err}') } + } + return paths } // get a new collection @@ -102,16 +171,15 @@ pub fn (mut tree Tree) add_collection(args_ CollectionNewArgs) ! { if args.name in tree.collections { if args.fail_on_error { return error('Collection with name ${args.name} already exits') - } - return - // TODO: report error + } + return } mut pp := pathlib.get_dir(path: args.path)! // will raise error if path doesn't exist mut new_collection := collection.new( - name: args.name - path: pp.path - heal: args.heal + name: args.name + path: pp.path + heal: args.heal fail_on_error: args.fail_on_error )! @@ -119,16 +187,18 @@ pub fn (mut tree Tree) add_collection(args_ CollectionNewArgs) ! { } // returns true if directory should be ignored while scanning -fn (tree Tree) is_ignored_dir(mut path pathlib.Path) !bool { +fn is_ignored_dir(path_ Path) !bool { + mut path := path_ if !path.is_dir() { return error('path is not a directory') } - - return path.name().starts_with('.') || path.name().starts_with('_') + name := path.name() + return name.starts_with('.') || name.starts_with('_') } -// gets collection name from .collection file or uses the directory name -fn (tree Tree) get_collection_name(mut path pathlib.Path) !string { +// gets collection name from .collection file +// if no name param, uses the directory name +fn get_collection_name(mut path Path) !string { mut collection_name := path.name() mut filepath := path.file_get('.collection')! @@ -145,12 +215,12 @@ fn (tree Tree) get_collection_name(mut path pathlib.Path) !string { return collection_name } -fn (tree Tree) is_collection_dir(path pathlib.Path) bool { +fn is_collection_dir(path Path) bool { return path.file_exists('.collection') } // moves .site file to .collection file -fn (tree Tree) move_site_to_collection(mut path pathlib.Path) ! { +fn move_site_to_collection(mut path Path) ! { collectionfilepath1 := path.extend_file('.site')! collectionfilepath2 := path.extend_file('.collection')! os.mv(collectionfilepath1.path, collectionfilepath2.path)! diff --git a/crystallib/core/openapi/readme.md b/crystallib/data/doctree/testdata/process_includes_test/col1/.collection similarity index 100% rename from crystallib/core/openapi/readme.md rename to crystallib/data/doctree/testdata/process_includes_test/col1/.collection diff --git a/crystallib/data/doctree/testdata/process_includes_test/col2/.collection b/crystallib/data/doctree/testdata/process_includes_test/col2/.collection new file mode 100644 index 000000000..e69de29bb diff --git a/crystallib/data/doctree/tree.v b/crystallib/data/doctree/tree.v index f75fe7c32..cefbf3b0d 100644 --- a/crystallib/data/doctree/tree.v +++ b/crystallib/data/doctree/tree.v @@ -47,7 +47,7 @@ pub fn new(args_ TreeArgsGet) !&Tree { mut args := args_ args.name = texttools.name_fix(args.name) mut t := Tree{ - name: args.name + name: args.name fail_on_error: args.fail_on_error } tree_set(t) diff --git a/crystallib/data/doctree/tree_test.v b/crystallib/data/doctree/tree_test.v index 7db86f357..a7e08d5d8 100644 --- a/crystallib/data/doctree/tree_test.v +++ b/crystallib/data/doctree/tree_test.v @@ -14,17 +14,17 @@ fn test_write_tree() { write_dir3 := pathlib.get_dir(path: '/tmp/tree_write3', empty: true)! // read tree1 - mut tree1 := new(name: doctree.tree_name)! - tree1.scan(path: doctree.collections_path)! + mut tree1 := new(name: tree_name)! + tree1.scan(path: collections_path)! tree1.export(destination: write_dir1.path)! // create tree2 from the written tree - mut tree2 := new(name: doctree.tree_name)! + mut tree2 := new(name: tree_name)! tree2.scan(path: write_dir1.path)! tree2.export(destination: write_dir2.path)! // write tree2 another time to compare the output of the two - mut tree3 := new(name: doctree.tree_name)! + mut tree3 := new(name: tree_name)! tree3.scan(path: write_dir2.path)! tree3.export(destination: write_dir3.path)! diff --git a/crystallib/data/jsonschema/model.v b/crystallib/data/jsonschema/model.v index b81bd822a..d57125950 100644 --- a/crystallib/data/jsonschema/model.v +++ b/crystallib/data/jsonschema/model.v @@ -22,10 +22,10 @@ pub mut: properties map[string]SchemaRef additional_properties SchemaRef @[json: 'additionalProperties'] required []string - ref string items Items defs map[string]SchemaRef one_of []SchemaRef @[json: 'oneOf'] + format string // todo: make fields optional upon the fixing of https://github.com/vlang/v/issues/18775 // from https://git.sr.ht/~emersion/go-jsonschema/tree/master/item/schema.go // Validation for numbers @@ -34,4 +34,5 @@ pub mut: exclusive_maximum int @[json: 'exclusiveMaximum'; omitempty] minimum int @[omitempty] exclusive_minimum int @[json: 'exclusiveMinimum'; omitempty] + enum_ []string @[json: 'enum'; omitempty] } diff --git a/crystallib/data/markdownparser/elements/base.v b/crystallib/data/markdownparser/elements/base.v index f08793686..7fd22d50d 100644 --- a/crystallib/data/markdownparser/elements/base.v +++ b/crystallib/data/markdownparser/elements/base.v @@ -134,7 +134,9 @@ pub fn (self DocBase) children() []Element { pub fn (mut self DocBase) process_children() !int { mut changes := 0 for mut element in self.children { - changes += element.process()! + changes += element.process() or { + return error('Failed to process child ${element.type_name}\n${err}') + } } return changes } diff --git a/crystallib/data/markdownparser/elements/element_action.v b/crystallib/data/markdownparser/elements/element_action.v index ffef5688d..39dc0c1fb 100644 --- a/crystallib/data/markdownparser/elements/element_action.v +++ b/crystallib/data/markdownparser/elements/element_action.v @@ -15,11 +15,7 @@ pub fn (mut self Action) process() !int { if self.processed { return 0 } - p := playbook.new(text: self.content)! - if p.actions.len != 1 { - return error('a single action is expected, but found ${p.actions.len}') - } - self.action = p.actions[0] + self.action = playbook.parse_single_action(self.content)! self.processed = true self.content = '' return 1 diff --git a/crystallib/data/markdownparser/elements/element_codeblock.v b/crystallib/data/markdownparser/elements/element_codeblock.v index a69020b05..eb5d700e0 100644 --- a/crystallib/data/markdownparser/elements/element_codeblock.v +++ b/crystallib/data/markdownparser/elements/element_codeblock.v @@ -14,13 +14,10 @@ pub fn (mut self Codeblock) process() !int { return 0 } // QUESTION: should we process actions here? - mut pb := playbook.new(text: self.content)! + // mut pb := playbook.new(text: self.content)! // if pb.actions.len > 0 { - // println('debugzo501') // for action in pb.actions { - // println('debugzo502 ${action.name}') // mut a := self.action_new(mut self.parent_doc(), '') - // println('debugzo503') // a.action = action // a.processed = true // a.content = action.heroscript() diff --git a/crystallib/data/markdownparser/factory.v b/crystallib/data/markdownparser/factory.v index 02f647e86..beb0606f2 100644 --- a/crystallib/data/markdownparser/factory.v +++ b/crystallib/data/markdownparser/factory.v @@ -16,7 +16,9 @@ pub: // get a parsed document, path is the path to the file, if not given content is needed pub fn new(args_ NewDocArgs) !elements.Doc { mut args := args_ - mut doc := elements.doc_new(collection_name: args.collection_name)! + mut doc := elements.doc_new(collection_name: args.collection_name) or { + return error('Failed create new doc ${args.collection_name}\n${err}') + } if args.path == '' { doc.content = args.content } else { @@ -29,6 +31,8 @@ pub fn new(args_ NewDocArgs) !elements.Doc { } } - parsers.parse_doc(mut doc)! + parsers.parse_doc(mut doc) or { + return error('Failed to parse doc ${args.path}\n${err}') + } return doc } diff --git a/crystallib/data/markdownparser/parsers/parse_doc.v b/crystallib/data/markdownparser/parsers/parse_doc.v index 6defc5e8a..005269549 100644 --- a/crystallib/data/markdownparser/parsers/parse_doc.v +++ b/crystallib/data/markdownparser/parsers/parse_doc.v @@ -6,7 +6,9 @@ import freeflowuniverse.crystallib.data.markdownparser.elements // DO NOT CHANGE THE WAY HOW THIS WORKS, THIS HAS BEEN DONE AS A STATEFUL PARSER BY DESIGN // THIS ALLOWS FOR EASY ADOPTIONS TO DIFFERENT RELIALITIES pub fn parse_doc(mut doc elements.Doc) ! { - mut parser := parser_line_new(mut doc)! + mut parser := parser_line_new(mut doc) or { + return error('Failed to parse line\n${err}') + } doc.type_name = 'doc' for { @@ -19,7 +21,9 @@ pub fn parse_doc(mut doc elements.Doc) ! { mut line := parser.line_current() trimmed_line := line.trim_space() - mut llast := parser.lastitem()! + mut llast := parser.lastitem() or { + return error('Failed to get last item\n${err}') + } // console.print_header('- line: ${llast.type_name} \'${line}\'') @@ -29,7 +33,9 @@ pub fn parse_doc(mut doc elements.Doc) ! { continue } - parser.ensure_last_is_paragraph()! + parser.ensure_last_is_paragraph() or { + return error('Failed to ensure last item is paragraph\n${err}') + } continue } @@ -39,7 +45,9 @@ pub fn parse_doc(mut doc elements.Doc) ! { parser.next() continue } - parser.next_start_lf()! + parser.next_start_lf() or { + return error('Failed to next start lf\n${err}') + } continue } @@ -179,5 +187,7 @@ pub fn parse_doc(mut doc elements.Doc) ! { // } // } - doc.process()! + doc.process() or { + return error('Failed to process doc\n${err}') + } } diff --git a/crystallib/data/ourdb/backend.v b/crystallib/data/ourdb/backend.v index aab5ecb3b..c62fef8f5 100644 --- a/crystallib/data/ourdb/backend.v +++ b/crystallib/data/ourdb/backend.v @@ -179,27 +179,27 @@ fn (mut db OurDB) get_prev_pos_(location Location) !Location { return db.lookup.location_new(prev_bytes)! } -// // delete zeros out the record at specified location -// fn (mut db OurDB) delete_(x u32, location Location) ! { -// if location.position == 0 { -// return error('Record not found') -// } - -// // Seek to position -// db.file.seek(i64(location.position), .start)! - -// // Read size first -// size_bytes := db.file.read_bytes(2) -// size := u16(size_bytes[0]) | (u16(size_bytes[1]) << 8) - -// // Write zeros for the entire record (header + data) -// zeros := []u8{len: int(size) + header_size, init: 0} -// db.file.seek(i64(location.position), .start)! -// db.file.write(zeros)! - -// // Clear lookup entry -// db.lookup.delete(x)! -// } +// delete zeros out the record at specified location +fn (mut db OurDB) delete_(x u32, location Location) ! { + if location.position == 0 { + return error('Record not found') + } + + // Seek to position + db.file.seek(i64(location.position), .start)! + + // Read size first + size_bytes := db.file.read_bytes(2) + size := u16(size_bytes[0]) | (u16(size_bytes[1]) << 8) + + // Write zeros for the entire record (header + data) + zeros := []u8{len: int(size) + header_size, init: 0} + db.file.seek(i64(location.position), .start)! + db.file.write(zeros)! + + // Clear lookup entry + db.lookup.delete(x)! +} // condense removes empty records and updates positions fn (mut db OurDB) condense() ! { diff --git a/crystallib/data/ourdb/db.v b/crystallib/data/ourdb/db.v index 5d04f602c..721abc734 100644 --- a/crystallib/data/ourdb/db.v +++ b/crystallib/data/ourdb/db.v @@ -17,10 +17,39 @@ import os // The data is stored with a CRC32 checksum for integrity verification // and maintains a linked list of previous values for history tracking // Returns the ID used (either x if specified, or auto-incremented if x=0) -pub fn (mut db OurDB) set(x u32, data []u8) !u32 { - location := db.lookup.get(x) or { Location{} } // Get location from lookup table if exists - db.set_(x, location, data)! - return db.lookup.set(x, location)! +@[params] +struct OurDBSetArgs { + id ?u32 + data []u8 @[required] +} + +pub fn (mut db OurDB) set(args OurDBSetArgs) !u32 { + if db.incremental_mode { + // if id points to an empty location, return an error + // else, overwrite data + if id := args.id { + // this is an update + location := db.lookup.get(id)! + if location.position == 0 { + return error('cannot set id for insertions when incremental mode is enabled') + } + + db.set_(id, location, args.data)! + db.lookup.set(id, location)! // TODO: maybe not needed + return id + } + + // this is an insert + id := db.lookup.get_next_id()! + db.set_(id, Location{}, args.data)! + return id + } + + // using key-value mode + id := args.id or { return error('id must be provided when incremental is disabled') } + location := db.lookup.get(id)! // Get location from lookup table if exists + db.set_(id, location, args.data)! + return id } // get retrieves data stored at the specified key position @@ -57,11 +86,9 @@ pub fn (mut db OurDB) get_history(x u32, depth u8) ![][]u8 { // This operation zeros out the record but maintains the space in the file // Use condense() to reclaim space from deleted records (happens in step after) pub fn (mut db OurDB) delete(x u32) ! { + location := db.lookup.get(x)! // Get location from lookup table + db.delete_(x, location)! db.lookup.delete(x)! - - // TODO: do we actually need to erase data? - // location := db.lookup.get(x)! // Get location from lookup table - // db.delete_(x, location)! } // close closes the database file diff --git a/crystallib/data/ourdb/db_test.v b/crystallib/data/ourdb/db_test.v index facaf33b9..27113af01 100644 --- a/crystallib/data/ourdb/db_test.v +++ b/crystallib/data/ourdb/db_test.v @@ -17,22 +17,22 @@ fn test_basic_operations() { // Test set and get test_data := 'Hello, World!'.bytes() - db.set(1, test_data)! + id := db.set(data: test_data)! - retrieved := db.get(1)! + retrieved := db.get(id)! assert retrieved == test_data // Test overwrite new_data := 'Updated data'.bytes() - db.set(1, new_data)! - retrieved2 := db.get(1)! + id2 := db.set(data: new_data)! + retrieved2 := db.get(id2)! assert retrieved2 == new_data } fn test_auto_increment() { mut db := new( - record_nr_max: 16777216 - 1 // max size of records - record_size_max: 1024 + record_nr_max: 10 // max size of records + record_size_max: 2 path: ourdb.test_dir )! @@ -42,16 +42,16 @@ fn test_auto_increment() { // Create 5 objects with no ID specified (x=0) mut ids := []u32{} - for i in 0..5 { + for i in 0 .. 5 { data := 'Object ${i + 1}'.bytes() - id := db.set(0, data)! + id := db.set(data: data)! ids << id } // Verify IDs are incremental assert ids.len == 5 - for i in 0..5 { - assert ids[i] == u32(i + 1) + for i in 0 .. 5 { + assert ids[i] == i // Verify data can be retrieved data := db.get(ids[i])! assert data == 'Object ${i + 1}'.bytes() @@ -63,6 +63,7 @@ fn test_history_tracking() { record_nr_max: 16777216 - 1 // max size of records record_size_max: 1024 path: ourdb.test_dir + incremental_mode: false )! defer { @@ -75,9 +76,9 @@ fn test_history_tracking() { data2 := 'Version 2'.bytes() data3 := 'Version 3'.bytes() - db.set(key, data1)! - db.set(key, data2)! - db.set(key, data3)! + db.set(id: key, data: data1)! + db.set(id: key, data: data2)! + db.set(id: key, data: data3)! // Get history with depth 3 history := db.get_history(key, 3)! @@ -92,6 +93,7 @@ fn test_delete_operation() { record_nr_max: 16777216 - 1 // max size of records record_size_max: 1024 path: ourdb.test_dir + incremental_mode: false )! defer { @@ -101,7 +103,7 @@ fn test_delete_operation() { // Set and then delete data test_data := 'Test data'.bytes() key := u32(1) - db.set(key, test_data)! + db.set(id: key, data: test_data)! // Verify data exists retrieved := db.get(key)! @@ -149,6 +151,7 @@ fn test_file_switching() { record_size_max: 1024 path: ourdb.test_dir file_size: 10 + incremental_mode: false )! defer { @@ -157,11 +160,11 @@ fn test_file_switching() { test_data1 := 'Test data'.bytes() key := u32(1) - db.set(key, test_data1)! + db.set(id: key, data: test_data1)! stat := os.stat('${db.path}/${db.last_used_file_nr}.db')! test_data2 := 'Test data 2222'.bytes() - db.set(u32(2), test_data2)! + db.set(id: u32(2), data: test_data2)! location := db.lookup.get(u32(2))! assert location.file_nr == 1 diff --git a/crystallib/data/ourdb/factory.v b/crystallib/data/ourdb/factory.v index ac7ce2e1c..30ac49b36 100644 --- a/crystallib/data/ourdb/factory.v +++ b/crystallib/data/ourdb/factory.v @@ -7,13 +7,15 @@ const mbyte_ = 1000000 // OurDB represents a binary database with variable-length records @[heap] pub struct OurDB { +mut: + lookup &LookupTable +pub: + path string // is the directory in which we will have the lookup db as well as all the backend + incremental_mode bool + file_size u32 = 500 * (1 << 20) // 500MB pub mut: - path string // is the directory in which we will have the lookup db as well as all the backend - lookup &LookupTable - file os.File - file_nr u16 // the file which is open - - file_size u32 = 500 * (1 << 20) // 500MB + file os.File + file_nr u16 // the file which is open last_used_file_nr u16 } @@ -26,6 +28,8 @@ pub: record_size_max u32 = 1024 * 4 // max size in bytes of a record, is 4 KB default file_size u32 = 500 * (1 << 20) // 500MB path string // directory where we will stor the DB + + incremental_mode bool = true } // new_memdb creates a new memory database with the given path and lookup table @@ -46,12 +50,18 @@ pub fn new(args OurDBConfig) !OurDB { keysize = 6 // will use multiple files } - mut l := new_lookup(size: args.record_nr_max, keysize: keysize)! + mut l := new_lookup( + size: args.record_nr_max + keysize: keysize + incremental_mode: args.incremental_mode + )! + os.mkdir_all(args.path)! mut db := OurDB{ path: args.path lookup: &l file_size: args.file_size + incremental_mode: args.incremental_mode } db.load()! diff --git a/crystallib/data/ourdb/lookup.v b/crystallib/data/ourdb/lookup.v index c02a75555..f6f7b4d1a 100644 --- a/crystallib/data/ourdb/lookup.v +++ b/crystallib/data/ourdb/lookup.v @@ -4,12 +4,17 @@ import os // LOOKUP table is link between the id and the posititon in a file with the data +const data_file_name = 'data' +const incremental_file_name = '.inc' + @[params] pub struct LookupConfig { pub: size u32 // size of the table keysize u8 // size of each entry in bytes (2-6), 6 means we store data over multiple files lookuppath string // if set, use disk-based lookup + + incremental_mode bool = true } pub struct LookupTable { @@ -17,7 +22,7 @@ pub struct LookupTable { lookuppath string mut: data []u8 - incremental u32 // tracks the last used incremental value + incremental ?u32 // points to next empty slot in the lookup table if incremental mode is enabled } // Method to create a new lookup table @@ -28,30 +33,55 @@ fn new_lookup(config LookupConfig) !LookupTable { } if config.lookuppath.len > 0 { - // For disk-based lookup, create empty file if it doesn't exist if !os.exists(config.lookuppath) { + os.mkdir_all(config.lookuppath)! + } + + // For disk-based lookup, create empty file if it doesn't exist + if !os.exists(os.join_path(config.lookuppath, ourdb.data_file_name)) { data := []u8{len: int(config.size * config.keysize), init: 0} - os.write_file(config.lookuppath, data.bytestr())! - // Create a separate file for storing the incremental value - os.write_file(config.lookuppath + '.inc', '0')! + os.write_file(os.join_path(config.lookuppath, ourdb.data_file_name), data.bytestr())! } - // Read the incremental value from file - inc_str := os.read_file(config.lookuppath + '.inc')! - incremental := inc_str.u32() + return LookupTable{ + // size: config.size data: []u8{} keysize: config.keysize lookuppath: config.lookuppath - incremental: incremental + incremental: get_incremental_info(config) } } return LookupTable{ + // size: config.size data: []u8{len: int(config.size * config.keysize), init: 0} keysize: config.keysize lookuppath: '' - incremental: 0 + incremental: get_incremental_info(config) + } +} + +fn get_incremental_info(config LookupConfig) ?u32 { + if !config.incremental_mode { + return none + } + + if config.lookuppath.len > 0 { + if !os.exists(os.join_path(config.lookuppath, ourdb.incremental_file_name)) { + // Create a separate file for storing the incremental value + os.write_file(os.join_path(config.lookuppath, ourdb.incremental_file_name), + '0') or { panic('failed to write .inc file: ${err}') } + } + + inc_str := os.read_file(os.join_path(config.lookuppath, ourdb.incremental_file_name)) or { + panic('failed to read .inc file: ${err}') + } + + incremental := inc_str.u32() + return incremental } + + return 0 } // Method to get value from a specific position @@ -59,7 +89,7 @@ fn (lut LookupTable) get(x u32) !Location { entry_size := int(lut.keysize) if lut.lookuppath.len > 0 { // Check file size first - file_size := os.file_size(lut.lookuppath) + file_size := os.file_size(lut.get_data_file_path()!) start_pos := x * entry_size if start_pos + entry_size > file_size { @@ -67,12 +97,11 @@ fn (lut LookupTable) get(x u32) !Location { } // Read directly from file for disk-based lookup - mut file := os.open(lut.lookuppath)! + mut file := os.open(lut.get_data_file_path()!)! defer { file.close() } - file.seek(start_pos, .start)! mut data := []u8{len: entry_size} - bytes_read := file.read(mut data)! + bytes_read := file.read_from(u64(start_pos), mut data)! if bytes_read < entry_size { return error('Incomplete read: expected ${entry_size} bytes but got ${bytes_read}') } @@ -87,19 +116,44 @@ fn (lut LookupTable) get(x u32) !Location { return lut.location_new(lut.data[start..start + entry_size])! } +fn (mut lut LookupTable) get_next_id() !u32 { + incremental := lut.incremental or { return error('lookup table not in incremental mode') } + + table_size := if lut.lookuppath.len > 0 { + u32(os.file_size(lut.get_data_file_path()!)) + } else { + u32(lut.data.len) + } + + if incremental * lut.keysize >= table_size { + return error('lookup table is full') + } + + return incremental +} + +fn (mut lut LookupTable) increment_index() ! { + mut incremental := lut.incremental or { return error('lookup table not in incremental mode') } + + incremental += 1 + lut.incremental = incremental + if lut.lookuppath.len > 0 { + os.write_file(lut.get_inc_file_path()!, incremental.str())! + } +} + // Method to set a value at a specific position -// Returns the ID used (either x if specified, or incremental if x=0) -fn (mut lut LookupTable) set(x u32, location Location) !u32 { +fn (mut lut LookupTable) set(x u32, location Location) ! { entry_size := int(lut.keysize) - + mut id := x - // Only increment if x is 0 - if x == 0 { - lut.incremental++ - id = lut.incremental - if lut.lookuppath.len > 0 { - // Update incremental value in file - os.write_file(lut.lookuppath + '.inc', lut.incremental.str())! + if incremental := lut.incremental { + if x == incremental { + lut.increment_index()! + } + + if x > incremental { + return error('cannot set id for insertions when incremental mode is enabled') } } @@ -108,26 +162,25 @@ fn (mut lut LookupTable) set(x u32, location Location) !u32 { // Check file size first file_size := os.file_size(lut.lookuppath) start_pos := id * entry_size - + data_file_path := lut.get_data_file_path()! if start_pos + entry_size > file_size { return error('Invalid write position: ${start_pos + entry_size} would exceed file size ${file_size}') } // Write directly to file for disk-based lookup - mut file := os.open_file(lut.lookuppath, 'w+')! - defer { + mut file := os.open_file(data_file_path, 'r+')! + defer { file.flush() - file.close() + file.close() } - file.seek(start_pos, .start)! - data := location.to_bytes()! - bytes_written := file.write(data[6 - entry_size..])! // Only write the required bytes based on keysize + bytes_written := file.write_to(u64(start_pos), data[(6 - entry_size)..])! // Only write the required bytes based on keysize if bytes_written < entry_size { return error('Incomplete write: expected ${entry_size} bytes but wrote ${bytes_written}') } - return id + + return } if id * u32(entry_size) >= u32(lut.data.len) { @@ -140,7 +193,6 @@ fn (mut lut LookupTable) set(x u32, location Location) !u32 { for i in 0 .. entry_size { lut.data[start + i] = bytes[6 - entry_size + i] // Only use the required bytes based on keysize } - return id } // Method to delete an entry (set bytes to 0) @@ -149,7 +201,7 @@ fn (mut lut LookupTable) delete(x u32) ! { if lut.lookuppath.len > 0 { // Check file size first - file_size := os.file_size(lut.lookuppath) + file_size := os.file_size(lut.get_data_file_path()!) start_pos := x * entry_size if start_pos + entry_size > file_size { @@ -157,15 +209,14 @@ fn (mut lut LookupTable) delete(x u32) ! { } // Write zeros directly to file for disk-based lookup - mut file := os.open_file(lut.lookuppath, 'w+')! - defer { + mut file := os.open_file(lut.get_data_file_path()!, 'r+')! + defer { file.flush() - file.close() + file.close() } - file.seek(start_pos, .start)! zeros := []u8{len: entry_size, init: 0} - bytes_written := file.write(zeros)! + bytes_written := file.write_to(u64(start_pos), zeros)! if bytes_written < entry_size { return error('Incomplete delete: expected ${entry_size} bytes but wrote ${bytes_written}') } @@ -186,12 +237,17 @@ fn (mut lut LookupTable) delete(x u32) ! { fn (lut LookupTable) export_data(path string) ! { if lut.lookuppath.len > 0 { // For disk-based lookup, copy both the main file and incremental value - os.cp(lut.lookuppath, path)! - os.cp(lut.lookuppath + '.inc', path + '.inc')! + os.cp(lut.get_data_file_path()!, os.join_path(path, ourdb.data_file_name))! + if _ := lut.incremental { + os.cp(lut.get_inc_file_path()!, os.join_path(path, ourdb.incremental_file_name))! + } return } - os.write_file(path, lut.data.bytestr())! - os.write_file(path + '.inc', lut.incremental.str())! + + os.write_file(os.join_path(path, ourdb.data_file_name), lut.data.bytestr())! + if incremental := lut.incremental { + os.write_file(os.join_path(path, ourdb.incremental_file_name), incremental.str())! + } } // Method to export the table in a sparse format @@ -201,10 +257,10 @@ fn (lut LookupTable) export_sparse(path string) ! { if lut.lookuppath.len > 0 { // For disk-based lookup, read the file in chunks - mut file := os.open(lut.lookuppath)! + mut file := os.open(lut.get_data_file_path()!)! defer { file.close() } - file_size := os.file_size(lut.lookuppath) + file_size := os.file_size(lut.get_data_file_path()!) mut buffer := []u8{len: entry_size} mut pos := u32(0) @@ -250,31 +306,41 @@ fn (lut LookupTable) export_sparse(path string) ! { } } } - os.write_file(path, output.bytestr())! + os.write_file(os.join_path(path, ourdb.data_file_name), output.bytestr())! // Also export the incremental value - os.write_file(path + '.inc', lut.incremental.str())! + if incremental := lut.incremental { + os.write_file(os.join_path(path, ourdb.incremental_file_name), incremental.str())! + } } // Method to import a lookup table from a file fn (mut lut LookupTable) import_data(path string) ! { if lut.lookuppath.len > 0 { // For disk-based lookup, copy both files - os.cp(path, lut.lookuppath)! - os.cp(path + '.inc', lut.lookuppath + '.inc')! - // Update the incremental value in memory - inc_str := os.read_file(path + '.inc')! - lut.incremental = inc_str.u32() + os.cp(os.join_path(path, ourdb.data_file_name), lut.get_data_file_path()!)! + + if _ := lut.incremental { + os.cp(os.join_path(path, ourdb.incremental_file_name), os.join_path(lut.lookuppath, + ourdb.incremental_file_name))! + // Update the incremental value in memory + inc_str := os.read_file(os.join_path(path, ourdb.incremental_file_name))! + println('inc_str: ${inc_str}') + lut.incremental = inc_str.u32() + } return } - lut.data = os.read_bytes(path)! - // Import the incremental value - inc_str := os.read_file(path + '.inc')! - lut.incremental = inc_str.u32() + + lut.data = os.read_bytes(os.join_path(path, ourdb.data_file_name))! + if _ := lut.incremental { + // Import the incremental value + inc_str := os.read_file(os.join_path(path, ourdb.incremental_file_name))! + lut.incremental = inc_str.u32() + } } // Method to import a sparse lookup table fn (mut lut LookupTable) import_sparse(path string) ! { - sparse_data := os.read_bytes(path)! + sparse_data := os.read_bytes(os.join_path(path, ourdb.data_file_name))! entry_size := int(lut.keysize) chunk_size := 4 + entry_size // 4 bytes for position + entry_size for value @@ -293,8 +359,24 @@ fn (mut lut LookupTable) import_sparse(path string) ! { lut.set(position, location)! } - - // Import the incremental value - inc_str := os.read_file(path + '.inc')! - lut.incremental = inc_str.u32() + + if _ := lut.incremental { + // Import the incremental value + inc_str := os.read_file(os.join_path(path, ourdb.incremental_file_name))! + lut.incremental = inc_str.u32() + } +} + +fn (lut LookupTable) get_data_file_path() !string { + if lut.lookuppath.len == 0 { + return error('lookup table is memory based') + } + + return os.join_path(lut.lookuppath, ourdb.data_file_name) +} + +fn (lut LookupTable) get_inc_file_path() !string { + _ := lut.incremental or { return error('incremental mode is disabled') } + + return os.join_path(lut.lookuppath, ourdb.incremental_file_name) } diff --git a/crystallib/data/ourdb/lookup_test.v b/crystallib/data/ourdb/lookup_test.v index b3eaffc4d..018425b3d 100644 --- a/crystallib/data/ourdb/lookup_test.v +++ b/crystallib/data/ourdb/lookup_test.v @@ -1,6 +1,7 @@ module ourdb import os +import rand const test_dir = '/tmp/lookuptest' @@ -17,6 +18,22 @@ fn testsuite_end() { } } +fn test_incremental() { + config := LookupConfig{ + size: 100 + keysize: 2 + } + mut lut := new_lookup(config)! + + assert lut.get_next_id()! == 0 + + lut.set(0, Location{ position: 23, file_nr: 0 })! + assert lut.get_next_id()! == 1 + + lut.set(1, Location{ position: 2, file_nr: 3 })! + assert lut.get_next_id()! == 2 +} + fn test_new_lookup() { // Test memory-based lookup config := LookupConfig{ @@ -53,7 +70,9 @@ fn test_set_get() { config := LookupConfig{ size: 100 keysize: 2 + incremental_mode: true } + mut lut := new_lookup(config)! // Test setting and getting values @@ -61,8 +80,10 @@ fn test_set_get() { position: 1234 file_nr: 0 } - id := lut.set(0, loc1)! - assert id == 1 // First auto-increment should be 1 + + id := lut.get_next_id()! + lut.set(id, loc1)! + result1 := lut.get(id)! assert result1.position == 1234 assert result1.file_nr == 0 @@ -72,9 +93,11 @@ fn test_set_get() { position: 5678 file_nr: 0 } - id2 := lut.set(5, loc2)! - assert id2 == 5 // Should return the specified ID - result2 := lut.get(5)! + + id2 := lut.get_next_id()! + lut.set(id2, loc2)! + assert id2 == 1 // Should return the specified ID + result2 := lut.get(id2)! assert result2.position == 5678 assert result2.file_nr == 0 @@ -91,7 +114,7 @@ fn test_disk_set_get() { config := LookupConfig{ size: 100 keysize: 2 - lookuppath: os.join_path(ourdb.test_dir, 'test.lut') + lookuppath: os.join_path(ourdb.test_dir, rand.string(4)) } mut lut := new_lookup(config)! @@ -100,8 +123,10 @@ fn test_disk_set_get() { position: 1234 file_nr: 0 } - id := lut.set(0, loc1)! - assert id == 1 // First auto-increment should be 1 + + id := lut.get_next_id()! + lut.set(id, loc1)! + assert id == 0 // First auto-increment should be 1 result1 := lut.get(id)! assert result1.position == 1234 assert result1.file_nr == 0 @@ -117,8 +142,10 @@ fn test_disk_set_get() { position: 5678 file_nr: 0 } - id2 := lut2.set(0, loc2)! - assert id2 == 2 // Should increment from previous value + + id2 := lut2.get_next_id()! + lut2.set(id2, loc2)! + assert id2 == 1 // Should increment from previous value } fn test_delete() { @@ -133,8 +160,11 @@ fn test_delete() { position: 1234 file_nr: 0 } - id := lut.set(0, loc1)! - assert id == 1 + + id := lut.get_next_id()! + lut.set(id, loc1)! + assert id == 0 + lut.delete(id)! result := lut.get(id)! assert result.position == 0 @@ -158,17 +188,23 @@ fn test_export_import() { position: 1234 file_nr: 0 } - id1 := lut.set(0, loc1)! - assert id1 == 1 + + id1 := lut.get_next_id()! + lut.set(id1, loc1)! + assert id1 == 0 + loc2 := Location{ position: 5678 file_nr: 0 } - id2 := lut.set(0, loc2)! - assert id2 == 2 + id2 := lut.get_next_id()! + lut.set(id2, loc2)! + assert id2 == 1 // Export and then import to new table export_path := os.join_path(ourdb.test_dir, 'export.lut') + os.mkdir(export_path)! + lut.export_data(export_path)! mut lut2 := new_lookup(config)! lut2.import_data(export_path)! @@ -182,14 +218,16 @@ fn test_export_import() { assert result2.file_nr == 0 // Verify incremental was imported - assert lut2.incremental == 2 + assert lut2.incremental! == 2 } fn test_export_import_sparse() { config := LookupConfig{ size: 100 keysize: 2 + incremental_mode: false } + mut lut := new_lookup(config)! // Set some values with gaps @@ -197,17 +235,21 @@ fn test_export_import_sparse() { position: 1234 file_nr: 0 } - id1 := lut.set(0, loc1)! - assert id1 == 1 + + id1 := u32(0) + lut.set(id1, loc1)! + loc2 := Location{ position: 5678 file_nr: 0 } - id2 := lut.set(50, loc2)! // Create a gap - assert id2 == 50 // Should use specified ID + id2 := u32(50) + lut.set(id2, loc2)! // Create a gap // Export and import sparse sparse_path := os.join_path(ourdb.test_dir, 'sparse.lut') + os.mkdir(sparse_path)! + lut.export_sparse(sparse_path)! mut lut2 := new_lookup(config)! lut2.import_sparse(sparse_path)! @@ -229,51 +271,62 @@ fn test_incremental_memory() { mut lut := new_lookup(config)! // Initial value should be 0 - assert lut.incremental == 0 + if incremental := lut.incremental { + assert incremental == 0 + } else { + assert false, 'incremental should have a value' + } // Set at x=0 should increment and return new ID loc1 := Location{ position: 1234 file_nr: 0 } - id1 := lut.set(0, loc1)! - assert id1 == 1 - assert lut.incremental == 1 + id1 := lut.get_next_id()! + lut.set(id1, loc1)! + assert id1 == 0 + assert lut.incremental! == 1 // Set at x=1 should not increment and return specified ID loc2 := Location{ position: 5678 file_nr: 0 } - id2 := lut.set(1, loc2)! + id2 := lut.get_next_id()! + lut.set(id2, loc2)! assert id2 == 1 - assert lut.incremental == 1 + assert lut.incremental! == 2 // Another set at x=0 should increment and return new ID loc3 := Location{ position: 9012 file_nr: 0 } - id3 := lut.set(0, loc3)! + + id3 := lut.get_next_id()! + lut.set(id3, loc3)! assert id3 == 2 - assert lut.incremental == 2 + assert lut.incremental! == 3 // Test persistence through export/import export_path := os.join_path(ourdb.test_dir, 'inc_export.lut') + os.mkdir(export_path)! + lut.export_data(export_path)! - + mut lut2 := new_lookup(config)! lut2.import_data(export_path)! - assert lut2.incremental == 2 + assert lut2.incremental! == 3 // Further operations should continue from last value loc4 := Location{ position: 3456 file_nr: 0 } - id4 := lut2.set(0, loc4)! + id4 := lut2.get_next_id()! + lut2.set(id4, loc4)! assert id4 == 3 - assert lut2.incremental == 3 + assert lut2.incremental! == 4 } fn test_incremental_disk() { @@ -285,9 +338,9 @@ fn test_incremental_disk() { mut lut := new_lookup(config)! // Initial value should be 0 - assert lut.incremental == 0 - assert os.exists(lut.lookuppath + '.inc') - inc_content := os.read_file(lut.lookuppath + '.inc')! + assert lut.incremental! == 0 + assert os.exists(lut.get_inc_file_path()!) + inc_content := os.read_file(lut.get_inc_file_path()!)! assert inc_content == '0' // Set at x=0 should increment @@ -295,10 +348,11 @@ fn test_incremental_disk() { position: 1234 file_nr: 0 } - id1 := lut.set(0, loc1)! - assert id1 == 1 - assert lut.incremental == 1 - inc_content1 := os.read_file(lut.lookuppath + '.inc')! + id1 := lut.get_next_id()! + lut.set(id1, loc1)! + assert id1 == 0 + assert lut.incremental! == 1 + inc_content1 := os.read_file(lut.get_inc_file_path()!)! assert inc_content1 == '1' // Set at x=1 should not increment @@ -306,26 +360,28 @@ fn test_incremental_disk() { position: 5678 file_nr: 0 } - id2 := lut.set(1, loc2)! + id2 := lut.get_next_id()! + lut.set(id2, loc2)! assert id2 == 1 - assert lut.incremental == 1 - inc_content2 := os.read_file(lut.lookuppath + '.inc')! - assert inc_content2 == '1' + assert lut.incremental! == 2 + inc_content2 := os.read_file(lut.get_inc_file_path()!)! + assert inc_content2 == '2' // Test persistence by creating new instance mut lut2 := new_lookup(config)! - assert lut2.incremental == 1 + assert lut2.incremental! == 2 // Further operations at x=0 should continue from last value loc3 := Location{ position: 9012 file_nr: 0 } - id3 := lut2.set(0, loc3)! + id3 := lut2.get_next_id()! + lut2.set(id3, loc3)! assert id3 == 2 - assert lut2.incremental == 2 - inc_content3 := os.read_file(lut.lookuppath + '.inc')! - assert inc_content3 == '2' + assert lut2.incremental! == 3 + inc_content3 := os.read_file(lut.get_inc_file_path()!)! + assert inc_content3 == '3' } fn test_multiple_sets() { @@ -337,30 +393,18 @@ fn test_multiple_sets() { // Set at x=0 five times mut ids := []u32{} - for i in 0..5 { + for i in 0 .. 5 { loc := Location{ - position: u32(1000 * (i + 1)) + position: 1000 * (i + 1) file_nr: 0 } - id := lut.set(0, loc)! - assert id == u32(i + 1) + id := lut.get_next_id()! + lut.set(id, loc)! + assert id == i ids << id } // Verify incremental is 5 - assert lut.incremental == 5 - assert ids == [u32(1), 2, 3, 4, 5] - - // Set at other positions should not affect incremental - for i in 1..5 { - loc := Location{ - position: u32(2000 * (i + 1)) - file_nr: 0 - } - id := lut.set(u32(i), loc)! - assert id == u32(i) - } - - // Incremental should still be 5 - assert lut.incremental == 5 + assert lut.incremental! == 5 + assert ids == [u32(0), 1, 2, 3, 4] } diff --git a/crystallib/develop/gittools/repository_utils.v b/crystallib/develop/gittools/repository_utils.v index 9693324c3..664c67c35 100644 --- a/crystallib/develop/gittools/repository_utils.v +++ b/crystallib/develop/gittools/repository_utils.v @@ -40,7 +40,10 @@ pub fn (repo GitRepo) get_path_of_url(url string) !string { } if repo_root_idx == -1 { - return error('Invalid URL format: Cannot find repository path') + // maybe default repo url (without src and blob) + return repo.get_path() or { + return error('Invalid URL format: Cannot find repository path') + } } // Ensure that the repository path starts after the branch diff --git a/crystallib/hero/README.md b/crystallib/hero/README.md new file mode 100644 index 000000000..e6eee4218 --- /dev/null +++ b/crystallib/hero/README.md @@ -0,0 +1,3 @@ +# Hero + +Crystallib module for hero specific development. \ No newline at end of file diff --git a/crystallib/hero/baobab/README.md b/crystallib/hero/baobab/README.md new file mode 100644 index 000000000..a0285edac --- /dev/null +++ b/crystallib/hero/baobab/README.md @@ -0,0 +1,51 @@ +# Base Object and Actor Backend + +This is Hero’s backend, designed around the concept of base objects and actors to enable modular, domain-specific operations. + +## Base Object + +Base objects are digital representations of real-world entities. Examples include projects, publications, books, stories (agile), and calendar events. These objects: + • Serve as the primary data units that actors operate on. + • Contain indexable fields for efficient retrieval. + • Share a common base class with attributes like: + • Name: The object’s identifier. + • Description: A brief summary of the object. + • Remarks: A list of additional notes or metadata. + +Base objects are stored, indexed, retrieved, and updated using OSIS (Object Storage and Indexing System). + +## Actor + +Actors are domain-specific operation handlers that work on base objects. For instance, a Project Manager Actor might manage operations on stories, sprints, or projects. + +Key Features of Actors: + • Domain-Specific Languages (DSLs): Actor methods form intuitive, logical DSLs for interacting with base objects. + • Specification-Driven: + • Actors are generated from specifications. + • Code written for actor methods can be parsed back into specifications. + • Code Generation: Specifications enable automated boilerplate code generation, reducing manual effort. + +## Modules + +### OSIS: Object Storage and Indexing System + +OSIS is a module designed for efficient storage and indexing of root objects based on specific fields. It enables seamless management of data across various backends, with built-in support for field-based filtering and searching. + +#### Key Components + +**Indexer:** +* Creates and manages SQL tables based on base object specifications. +* Enables indexing of specific fields, making them searchable and filterable. + +**Storer**: +* Handles actual data storage in different databases. +* Supports diverse encoding and encryption methods for secure data management. + +By integrating OSIS, the backend achieves both high-performance data querying and flexible, secure storage solutions. + +### Example Actor Module + +The Example Actor module is a reference and testable example of a generated actor within Baobab. It demonstrates the structure of actor modules generated from specifications and can also be parsed back into specifications. This module serves two key purposes: + +1. Acts as a reference for developers working on Baobab to understand and program against actor specifications. +2. Provides a compilable, generatable module for testing and validating Baobab’s code generation tools. \ No newline at end of file diff --git a/crystallib/baobab/_archive/backend/README.md b/crystallib/hero/baobab/_archive/backend/README.md similarity index 100% rename from crystallib/baobab/_archive/backend/README.md rename to crystallib/hero/baobab/_archive/backend/README.md diff --git a/crystallib/baobab/_archive/backend/backend.v b/crystallib/hero/baobab/_archive/backend/backend.v similarity index 98% rename from crystallib/baobab/_archive/backend/backend.v rename to crystallib/hero/baobab/_archive/backend/backend.v index e0d8a0bde..fdc685cca 100644 --- a/crystallib/baobab/_archive/backend/backend.v +++ b/crystallib/hero/baobab/_archive/backend/backend.v @@ -3,7 +3,7 @@ module backend import os import db.sqlite import db.pg -import freeflowuniverse.crystallib.core.dbfs +import freeflowuniverse.crystallib.data.dbfs import freeflowuniverse.crystallib.data.encoderhero pub struct Backend { diff --git a/crystallib/baobab/_archive/backend/backend_generic.v b/crystallib/hero/baobab/_archive/backend/backend_generic.v similarity index 100% rename from crystallib/baobab/_archive/backend/backend_generic.v rename to crystallib/hero/baobab/_archive/backend/backend_generic.v diff --git a/crystallib/baobab/_archive/backend/database.v b/crystallib/hero/baobab/_archive/backend/database.v similarity index 100% rename from crystallib/baobab/_archive/backend/database.v rename to crystallib/hero/baobab/_archive/backend/database.v diff --git a/crystallib/baobab/_archive/backend/identifier.v b/crystallib/hero/baobab/_archive/backend/identifier.v similarity index 100% rename from crystallib/baobab/_archive/backend/identifier.v rename to crystallib/hero/baobab/_archive/backend/identifier.v diff --git a/crystallib/baobab/_archive/backend/indexer.v b/crystallib/hero/baobab/_archive/backend/indexer.v similarity index 100% rename from crystallib/baobab/_archive/backend/indexer.v rename to crystallib/hero/baobab/_archive/backend/indexer.v diff --git a/crystallib/baobab/_archive/backend/indexer_generic.v b/crystallib/hero/baobab/_archive/backend/indexer_generic.v similarity index 100% rename from crystallib/baobab/_archive/backend/indexer_generic.v rename to crystallib/hero/baobab/_archive/backend/indexer_generic.v diff --git a/crystallib/baobab/_archive/backend/indexer_generic_test.v b/crystallib/hero/baobab/_archive/backend/indexer_generic_test.v similarity index 100% rename from crystallib/baobab/_archive/backend/indexer_generic_test.v rename to crystallib/hero/baobab/_archive/backend/indexer_generic_test.v diff --git a/crystallib/baobab/_archive/backend/indexer_test.v b/crystallib/hero/baobab/_archive/backend/indexer_test.v similarity index 100% rename from crystallib/baobab/_archive/backend/indexer_test.v rename to crystallib/hero/baobab/_archive/backend/indexer_test.v diff --git a/crystallib/baobab/_archive/backend/simple_sqlite/example.sqlite b/crystallib/hero/baobab/_archive/backend/simple_sqlite/example.sqlite similarity index 100% rename from crystallib/baobab/_archive/backend/simple_sqlite/example.sqlite rename to crystallib/hero/baobab/_archive/backend/simple_sqlite/example.sqlite diff --git a/crystallib/baobab/_archive/backend/simple_sqlite/example.v b/crystallib/hero/baobab/_archive/backend/simple_sqlite/example.v similarity index 100% rename from crystallib/baobab/_archive/backend/simple_sqlite/example.v rename to crystallib/hero/baobab/_archive/backend/simple_sqlite/example.v diff --git a/crystallib/baobab/_archive/backend/storer copy.v b/crystallib/hero/baobab/_archive/backend/storer copy.v similarity index 100% rename from crystallib/baobab/_archive/backend/storer copy.v rename to crystallib/hero/baobab/_archive/backend/storer copy.v diff --git a/crystallib/baobab/_archive/backend/storer.v b/crystallib/hero/baobab/_archive/backend/storer.v similarity index 100% rename from crystallib/baobab/_archive/backend/storer.v rename to crystallib/hero/baobab/_archive/backend/storer.v diff --git a/crystallib/baobab/code/README.md b/crystallib/hero/baobab/_archive/code/README.md similarity index 100% rename from crystallib/baobab/code/README.md rename to crystallib/hero/baobab/_archive/code/README.md diff --git a/crystallib/baobab/code/model.v b/crystallib/hero/baobab/_archive/code/model.v similarity index 100% rename from crystallib/baobab/code/model.v rename to crystallib/hero/baobab/_archive/code/model.v diff --git a/crystallib/baobab/code/read.v b/crystallib/hero/baobab/_archive/code/read.v similarity index 82% rename from crystallib/baobab/code/read.v rename to crystallib/hero/baobab/_archive/code/read.v index ec0c8430b..6c6fb6e63 100644 --- a/crystallib/baobab/code/read.v +++ b/crystallib/hero/baobab/_archive/code/read.v @@ -3,7 +3,7 @@ module code import freeflowuniverse.crystallib.core.pathlib {Path} import freeflowuniverse.crystallib.core.texttools import freeflowuniverse.crystallib.core.codeparser -import freeflowuniverse.crystallib.core.codemodel {Module, CodeFile, Function, Struct} +import freeflowuniverse.crystallib.core.codemodel {Module, VFile, Function, Struct} // read reads an actor from a given v module pub fn read(actor_path string) !Actor { @@ -29,7 +29,7 @@ pub fn module_to_actor(mod Module) !Actor { return actor } -fn file_to_base_object(file CodeFile) !BaseObject { +fn file_to_base_object(file VFile) !BaseObject { object_name := texttools.name_fix_snake_to_pascal(file.name.all_after('model_').all_before('.')) object_structure := file.structs().filter(it.name == object_name)[0] return BaseObject { @@ -37,6 +37,6 @@ fn file_to_base_object(file CodeFile) !BaseObject { } } -fn file_to_actor_methods(file CodeFile) ![]ActorMethod { +fn file_to_actor_methods(file VFile) ![]ActorMethod { return file.functions().map(ActorMethod{name: it.name, func: it}) } \ No newline at end of file diff --git a/crystallib/baobab/code/read_test.v b/crystallib/hero/baobab/_archive/code/read_test.v similarity index 100% rename from crystallib/baobab/code/read_test.v rename to crystallib/hero/baobab/_archive/code/read_test.v diff --git a/crystallib/baobab/code/templates/cli.v.template b/crystallib/hero/baobab/_archive/code/templates/cli.v.template similarity index 100% rename from crystallib/baobab/code/templates/cli.v.template rename to crystallib/hero/baobab/_archive/code/templates/cli.v.template diff --git a/crystallib/baobab/code/templates/playground.v.template b/crystallib/hero/baobab/_archive/code/templates/playground.v.template similarity index 100% rename from crystallib/baobab/code/templates/playground.v.template rename to crystallib/hero/baobab/_archive/code/templates/playground.v.template diff --git a/crystallib/baobab/code/testdata/testactor/model_another_object.v b/crystallib/hero/baobab/_archive/code/testdata/testactor/model_another_object.v similarity index 100% rename from crystallib/baobab/code/testdata/testactor/model_another_object.v rename to crystallib/hero/baobab/_archive/code/testdata/testactor/model_another_object.v diff --git a/crystallib/baobab/code/testdata/testactor/model_base_object.v b/crystallib/hero/baobab/_archive/code/testdata/testactor/model_base_object.v similarity index 100% rename from crystallib/baobab/code/testdata/testactor/model_base_object.v rename to crystallib/hero/baobab/_archive/code/testdata/testactor/model_base_object.v diff --git a/crystallib/baobab/code/testdata/testactor/testactor_another_object.v b/crystallib/hero/baobab/_archive/code/testdata/testactor/testactor_another_object.v similarity index 100% rename from crystallib/baobab/code/testdata/testactor/testactor_another_object.v rename to crystallib/hero/baobab/_archive/code/testdata/testactor/testactor_another_object.v diff --git a/crystallib/baobab/code/testdata/testactor/testactor_base_object.v b/crystallib/hero/baobab/_archive/code/testdata/testactor/testactor_base_object.v similarity index 100% rename from crystallib/baobab/code/testdata/testactor/testactor_base_object.v rename to crystallib/hero/baobab/_archive/code/testdata/testactor/testactor_base_object.v diff --git a/crystallib/baobab/code/testdata/testactor/testactor_base_object_custom.v b/crystallib/hero/baobab/_archive/code/testdata/testactor/testactor_base_object_custom.v similarity index 100% rename from crystallib/baobab/code/testdata/testactor/testactor_base_object_custom.v rename to crystallib/hero/baobab/_archive/code/testdata/testactor/testactor_base_object_custom.v diff --git a/crystallib/baobab/code/to_openrpc.v b/crystallib/hero/baobab/_archive/code/to_openrpc.v similarity index 100% rename from crystallib/baobab/code/to_openrpc.v rename to crystallib/hero/baobab/_archive/code/to_openrpc.v diff --git a/crystallib/baobab/code/to_openrpc_test.v b/crystallib/hero/baobab/_archive/code/to_openrpc_test.v similarity index 100% rename from crystallib/baobab/code/to_openrpc_test.v rename to crystallib/hero/baobab/_archive/code/to_openrpc_test.v diff --git a/crystallib/baobab/code/write.v b/crystallib/hero/baobab/_archive/code/write.v similarity index 84% rename from crystallib/baobab/code/write.v rename to crystallib/hero/baobab/_archive/code/write.v index c144ce343..eb1ef439d 100644 --- a/crystallib/baobab/code/write.v +++ b/crystallib/hero/baobab/_archive/code/write.v @@ -3,7 +3,7 @@ module code // import freeflowuniverse.crystallib.core.pathlib {Path} // import freeflowuniverse.crystallib.core.texttools // import freeflowuniverse.crystallib.core.codeparser -import freeflowuniverse.crystallib.core.codemodel {Module, CodeFile, Function, Struct} +import freeflowuniverse.crystallib.core.codemodel {Module, VFile, Function, Struct} // read reads an actor from a given v module pub fn (actor Actor) write(actor_path string) ! { @@ -16,7 +16,7 @@ pub fn (actor Actor) to_module () !Module { name: actor.name } - mut files := []CodeFile{} + mut files := []VFile{} for object in actor.objects { files << object.to_code_file()! } @@ -45,15 +45,15 @@ pub fn (actor Actor) to_module () !Module { return mod } -fn (object BaseObject) to_code_file() !CodeFile { +fn (object BaseObject) to_code_file() !VFile { // object_name := texttools.name_fix_snake_to_pascal(file.name.all_after('model_').all_before('.')) // object_structure := file.structs().filter(it.name == object_name)[0] - return CodeFile { + return VFile { name: 'model_${object.structure.name}' items: [object.structure] } } -// fn file_to_actor_methods(file CodeFile) ![]ActorMethod { +// fn file_to_actor_methods(file VFile) ![]ActorMethod { // return file.functions().map(ActorMethod{name: it.name, func: it}) // } \ No newline at end of file diff --git a/crystallib/baobab/code/write_test.v b/crystallib/hero/baobab/_archive/code/write_test.v similarity index 100% rename from crystallib/baobab/code/write_test.v rename to crystallib/hero/baobab/_archive/code/write_test.v diff --git a/crystallib/hero/baobab/_archive/processor/processor.v b/crystallib/hero/baobab/_archive/processor/processor.v new file mode 100644 index 000000000..ba67de82f --- /dev/null +++ b/crystallib/hero/baobab/_archive/processor/processor.v @@ -0,0 +1,41 @@ +module processor + +import freeflowuniverse.crystallib.clients.redisclient + +// Processor struct for managing procedure calls +pub struct Processor { +pub mut: + rpc redisclient.RedisRpc // Redis RPC mechanism +} + +// Parameters for processing a procedure call +@[params] +pub struct ProcessParams { +pub: + timeout int // Timeout in seconds +} + +// Process the procedure call +pub fn (mut p Processor) process(call ProcedureCall, params ProcessParams) !ProcedureResponse { + // Use RedisRpc's `call` to send the call and wait for the response + response_data := p.rpc.call(redisclient.RPCArgs{ + cmd: call.method + data: call.params + timeout: u64(params.timeout * 1000) // Convert seconds to milliseconds + wait: true + }) or { + // TODO: check error type + return ProcedureResponse{ + error: err.msg() + } + // return ProcedureError{ + // reason: .timeout + // } + } + + println('resp data ${response_data}') + + return ProcedureResponse{ + result: response_data + } +} \ No newline at end of file diff --git a/crystallib/hero/baobab/_archive/processor/rpc.v b/crystallib/hero/baobab/_archive/processor/rpc.v new file mode 100644 index 000000000..9317d519a --- /dev/null +++ b/crystallib/hero/baobab/_archive/processor/rpc.v @@ -0,0 +1,118 @@ +module processor + +import os +import time +import veb +import json +import x.json2 {Any} +import net.http +import freeflowuniverse.crystallib.data.jsonschema {Schema} +import freeflowuniverse.crystallib.web.openapi {Server, Context, Request, Response} +// import freeflowuniverse.crystallib.hero.processor {Processor, ProcedureCall, ProcedureResponse, ProcessParams} +import freeflowuniverse.crystallib.clients.redisclient + +// pub struct Handler { +// pub mut: +// client actor.Client +// } + +// fn (mut handler Handler) handle(request Request) !Response { +// // Convert incoming OpenAPI request to a procedure call +// mut params := []string{} + +// if request.arguments.len > 0 { +// params = request.arguments.values().map(it.str()).clone() +// } + +// if request.body != '' { +// params << request.body +// } + +// if request.parameters.len != 0 { +// mut param_map := map[string]Any{} // Store parameters with correct types + +// for param_name, param_value in request.parameters { +// operation_param := request.operation.parameters.filter(it.name == param_name) +// if operation_param.len > 0 { +// param_schema := operation_param[0].schema as Schema +// param_type := param_schema.typ +// param_format := param_schema.format + +// // Convert parameter value to corresponding type +// match param_type { +// 'integer' { +// match param_format { +// 'int32' { +// param_map[param_name] = param_value.int() // Convert to int +// } +// 'int64' { +// param_map[param_name] = param_value.i64() // Convert to i64 +// } +// else { +// param_map[param_name] = param_value.int() // Default to int +// } +// } +// } +// 'string' { +// param_map[param_name] = param_value // Already a string +// } +// 'boolean' { +// param_map[param_name] = param_value.bool() // Convert to bool +// } +// 'number' { +// match param_format { +// 'float' { +// param_map[param_name] = param_value.f32() // Convert to float +// } +// 'double' { +// param_map[param_name] = param_value.f64() // Convert to double +// } +// else { +// param_map[param_name] = param_value.f64() // Default to double +// } +// } +// } +// else { +// param_map[param_name] = param_value // Leave as string for unknown types +// } +// } +// } else { +// // If the parameter is not defined in the OpenAPI operation, skip or log it +// println('Unknown parameter: $param_name') +// } +// } + +// // Encode the parameter map to JSON if needed +// params << json.encode(param_map.str()) +// } + +// call := ProcedureCall{ +// method: request.operation.operation_id +// params: "[${params.join(',')}]" // Keep as a string since ProcedureCall expects a string +// } + +// // Process the procedure call +// procedure_response := handler.client.dialogue( +// call, +// ProcessParams{ +// timeout: 30 // Set timeout in seconds +// } +// ) or { +// // Handle ProcedureError +// if err is ProcedureError { +// return Response{ +// status: http.status_from_int(err.code()) // Map ProcedureError reason to HTTP status code +// body: json.encode({ +// 'error': err.msg() +// }) +// } +// } +// return error('Unexpected error: $err') +// } + +// // Convert returned procedure response to OpenAPI response +// return Response{ +// status: http.Status.ok // Assuming success if no error +// body: procedure_response.result +// } +// } diff --git a/crystallib/baobab/representation/html.v b/crystallib/hero/baobab/_archive/representation/html.v similarity index 100% rename from crystallib/baobab/representation/html.v rename to crystallib/hero/baobab/_archive/representation/html.v diff --git a/crystallib/baobab/representation/readme.md b/crystallib/hero/baobab/_archive/representation/readme.md similarity index 100% rename from crystallib/baobab/representation/readme.md rename to crystallib/hero/baobab/_archive/representation/readme.md diff --git a/crystallib/baobab/seeds/finance/budget.v b/crystallib/hero/baobab/_archive/seeds/finance/budget.v similarity index 100% rename from crystallib/baobab/seeds/finance/budget.v rename to crystallib/hero/baobab/_archive/seeds/finance/budget.v diff --git a/crystallib/baobab/seeds/populate.vsh b/crystallib/hero/baobab/_archive/seeds/populate.vsh similarity index 100% rename from crystallib/baobab/seeds/populate.vsh rename to crystallib/hero/baobab/_archive/seeds/populate.vsh diff --git a/crystallib/baobab/seeds/project/story.v b/crystallib/hero/baobab/_archive/seeds/project/story.v similarity index 100% rename from crystallib/baobab/seeds/project/story.v rename to crystallib/hero/baobab/_archive/seeds/project/story.v diff --git a/crystallib/baobab/seeds/schedule/calendar.v b/crystallib/hero/baobab/_archive/seeds/schedule/calendar.v similarity index 100% rename from crystallib/baobab/seeds/schedule/calendar.v rename to crystallib/hero/baobab/_archive/seeds/schedule/calendar.v diff --git a/crystallib/baobab/seeds/schedule/event.v b/crystallib/hero/baobab/_archive/seeds/schedule/event.v similarity index 100% rename from crystallib/baobab/seeds/schedule/event.v rename to crystallib/hero/baobab/_archive/seeds/schedule/event.v diff --git a/crystallib/baobab/view/README.md b/crystallib/hero/baobab/_archive/view/README.md similarity index 100% rename from crystallib/baobab/view/README.md rename to crystallib/hero/baobab/_archive/view/README.md diff --git a/crystallib/baobab/view/html.v b/crystallib/hero/baobab/_archive/view/html.v similarity index 100% rename from crystallib/baobab/view/html.v rename to crystallib/hero/baobab/_archive/view/html.v diff --git a/crystallib/hero/baobab/action/action.v b/crystallib/hero/baobab/action/action.v new file mode 100644 index 000000000..ddf4419bd --- /dev/null +++ b/crystallib/hero/baobab/action/action.v @@ -0,0 +1,95 @@ +module action + +import crypto.blake2b +import freeflowuniverse.crystallib.data.paramsparser +import freeflowuniverse.crystallib.core.texttools +// import freeflowuniverse.crystallib.core.smartid + +pub struct Action { +pub mut: + id int + cid string + name string + actor string + priority int = 10 // 0 is highest, do 10 as default + params paramsparser.Params + result paramsparser.Params // can be used to remember outputs + // run bool = true // certain actions can be defined but meant to be executed directly + actiontype ActionType = .sal + comments string + done bool // if done then no longer need to process +} + +pub enum ActionType { + unknown + dal + sal + wal + macro +} + +pub fn (action Action) str() string { + mut out := action.heroscript() + if !action.result.empty() { + out += '\n\nResult:\n' + out += texttools.indent(action.result.heroscript(), ' ') + } + return out +} + +// serialize to heroscript +pub fn (action Action) heroscript() string { + mut out := '' + if action.comments.len > 0 { + out += texttools.indent(action.comments, '// ') + } + if action.actiontype == .sal { + out += '!!' + } else if action.actiontype == .macro { + out += '!!!' + } else { + panic('only action sal and macro supported for now,\n${action}') + } + + if action.actor != '' { + out += '${action.actor}.' + } + out += '${action.name} ' + if action.id > 0 { + out += 'id:${action.id} ' + } + if !action.params.empty() { + heroscript := action.params.heroscript() + heroscript_lines := heroscript.split_into_lines() + out += heroscript_lines[0] + '\n' + for line in heroscript_lines[1..] { + out += ' ' + line + '\n' + } + } + return out +} + +// return list of names . +// the names are normalized (no special chars, lowercase, ... ) +pub fn (action Action) names() []string { + mut names := []string{} + for name in action.name.split('.') { + names << texttools.name_fix(name) + } + return names +} + +pub enum ActionState { + init // first state + next // will continue with next steps + restart + error + done // means we don't process the next ones +} + +// get hash from the action, should always be the same for the same action +pub fn (action Action) hashkey() string { + txt := action.heroscript() + bs := blake2b.sum160(txt.bytes()) + return bs.hex() +} diff --git a/crystallib/hero/baobab/action/error.v b/crystallib/hero/baobab/action/error.v new file mode 100644 index 000000000..02b52eb0d --- /dev/null +++ b/crystallib/hero/baobab/action/error.v @@ -0,0 +1,33 @@ +module action + +// Error struct for error handling +pub struct ActionError { + reason ErrorReason +} + +// Enum for different error reasons +pub enum ErrorReason { + timeout + serialization_failed + deserialization_failed + enqueue_failed +} + +pub fn (err ActionError) code() int { + return match err.reason { + .timeout { 408 } // HTTP 408 Request Timeout + .serialization_failed { 500 } // HTTP 500 Internal Server Error + .deserialization_failed { 500 } // HTTP 500 Internal Server Error + .enqueue_failed { 503 } // HTTP 503 Service Unavailable + } +} + +pub fn (err ActionError) msg() string { + explanation := match err.reason { + .timeout { 'The procedure call timed out.' } + .serialization_failed { 'Failed to serialize the procedure call.' } + .deserialization_failed { 'Failed to deserialize the procedure response.' } + .enqueue_failed { 'Failed to enqueue the procedure response.' } + } + return 'Procedure failed: $explanation' +} diff --git a/crystallib/hero/baobab/action/procedure.v b/crystallib/hero/baobab/action/procedure.v new file mode 100644 index 000000000..89f029857 --- /dev/null +++ b/crystallib/hero/baobab/action/procedure.v @@ -0,0 +1,15 @@ +module action + +// ProcedureResponse struct representing the result of a procedure call +pub struct ProcedureResponse { +pub: + result string // Response data + error string // Internal error message (if any) +} + +// Parameters for processing a procedure call +@[params] +pub struct ProcessParams { +pub: + timeout int // Timeout in seconds +} \ No newline at end of file diff --git a/crystallib/hero/baobab/action/reflection_openapi.v b/crystallib/hero/baobab/action/reflection_openapi.v new file mode 100644 index 000000000..548584b6f --- /dev/null +++ b/crystallib/hero/baobab/action/reflection_openapi.v @@ -0,0 +1,89 @@ +module action + +import json +import os +import time +import veb +import x.json2 {Any} +import net.http +import freeflowuniverse.crystallib.data.jsonschema {Schema} +// import freeflowuniverse.crystallib.hero.processor {Processor, ProcedureCall, ProcedureResponse, ProcessParams} +import freeflowuniverse.crystallib.clients.redisclient +import freeflowuniverse.crystallib.web.openapi {Request} + +pub fn openapi_request_to_action(request Request) Action { + // Convert incoming OpenAPI request to a procedure call + mut params := []string{} + + if request.arguments.len > 0 { + params = request.arguments.values().map(it.str()).clone() + } + + if request.body != '' { + params << request.body + } + + if request.parameters.len != 0 { + mut param_map := map[string]Any{} // Store parameters with correct types + + for param_name, param_value in request.parameters { + operation_param := request.operation.parameters.filter(it.name == param_name) + if operation_param.len > 0 { + param_schema := operation_param[0].schema as Schema + param_type := param_schema.typ + param_format := param_schema.format + + // Convert parameter value to corresponding type + match param_type { + 'integer' { + match param_format { + 'int32' { + param_map[param_name] = param_value.int() // Convert to int + } + 'int64' { + param_map[param_name] = param_value.i64() // Convert to i64 + } + else { + param_map[param_name] = param_value.int() // Default to int + } + } + } + 'string' { + param_map[param_name] = param_value // Already a string + } + 'boolean' { + param_map[param_name] = param_value.bool() // Convert to bool + } + 'number' { + match param_format { + 'float' { + param_map[param_name] = param_value.f32() // Convert to float + } + 'double' { + param_map[param_name] = param_value.f64() // Convert to double + } + else { + param_map[param_name] = param_value.f64() // Default to double + } + } + } + else { + param_map[param_name] = param_value // Leave as string for unknown types + } + } + } else { + // If the parameter is not defined in the OpenAPI operation, skip or log it + println('Unknown parameter: $param_name') + } + } + + // Encode the parameter map to JSON if needed + params << json.encode(param_map.str()) + } + + call := Action{ + method: request.operation.operation_id + params: "[${params.join(',')}]" // Keep as a string since ProcedureCall expects a string + } + return call +} \ No newline at end of file diff --git a/crystallib/hero/baobab/actor/actor.v b/crystallib/hero/baobab/actor/actor.v new file mode 100644 index 000000000..ea4732ac0 --- /dev/null +++ b/crystallib/hero/baobab/actor/actor.v @@ -0,0 +1,31 @@ +module actor + +import freeflowuniverse.crystallib.clients.redisclient +import time + +pub interface IActor { + name string +mut: + handle(string, string) !string +} + +pub struct Actor { +pub: + name string +} + +pub fn new(name string) Actor { + return Actor{name} +} + +// Actor listens to the Redis queue for method invocations +pub fn (mut actor IActor) run() ! { + mut redis := redisclient.new('localhost:6379') or { panic(err) } + mut rpc := redis.rpc_get(actor.name) + + println('Actor started and listening for tasks...') + for { + rpc.process(actor.handle)! + time.sleep(time.millisecond * 100) // Prevent CPU spinning + } +} diff --git a/crystallib/hero/baobab/actor/client.v b/crystallib/hero/baobab/actor/client.v new file mode 100644 index 000000000..d243037c7 --- /dev/null +++ b/crystallib/hero/baobab/actor/client.v @@ -0,0 +1,70 @@ +module actor + +import json +import freeflowuniverse.crystallib.clients.redisclient +import freeflowuniverse.crystallib.hero.baobab.action { ProcedureCall, ProcedureResponse } + +// Processor struct for managing procedure calls +pub struct Client { +pub mut: + rpc redisclient.RedisRpc // Redis RPC mechanism +} + +// Parameters for processing a procedure call +@[params] +pub struct Params { +pub: + timeout int // Timeout in seconds +} + +pub struct ClientConfig { +pub: + redis_url string // url to redis server running + redis_queue string // name of redis queue +} + +pub fn new_client(config ClientConfig) !Client { + mut redis := redisclient.new(config.redis_url)! + mut rpc_q := redis.rpc_get(config.redis_queue) + + return Client{ + rpc: rpc_q + } +} + +// Process the procedure call +pub fn (mut p Client) monologue(call ProcedureCall, params Params) ! { + // Use RedisRpc's `call` to send the call and wait for the response + response_data := p.rpc.call(redisclient.RPCArgs{ + cmd: call.method + data: call.params + timeout: u64(params.timeout * 1000) // Convert seconds to milliseconds + wait: true + })! + // TODO: check error type +} + +// Process the procedure call +pub fn (mut p Client) call_to_action (action Procedure, params Params) !ProcedureResponse { + // Use RedisRpc's `call` to send the call and wait for the response + response_data := p.rpc.call(redisclient.RPCArgs{ + cmd: call.method + data: call.params + timeout: u64(params.timeout * 1000) // Convert seconds to milliseconds + wait: true + }) or { + // TODO: check error type + return ProcedureResponse{ + error: err.msg() + } + // return ProcedureError{ + // reason: .timeout + // } + } + + println('resp data ${response_data}') + + return ProcedureResponse{ + result: response_data + } +} diff --git a/crystallib/hero/baobab/actor/proxy_openapi.v b/crystallib/hero/baobab/actor/proxy_openapi.v new file mode 100644 index 000000000..9964eeffe --- /dev/null +++ b/crystallib/hero/baobab/actor/proxy_openapi.v @@ -0,0 +1,78 @@ +module actor + +import veb +import freeflowuniverse.crystallib.web.openapi { Context, Controller, OpenAPI, Request, Response } +import freeflowuniverse.crystallib.hero.baobab.action { ProcedureError } +import os +import time +import json +import x.json2 +import net.http +import freeflowuniverse.crystallib.data.jsonschema +// import freeflowuniverse.crystallib.hero.processor {Processor, ProcedureCall, ProcedureResponse, ProcessParams} +import freeflowuniverse.crystallib.clients.redisclient + +pub struct OpenAPIProxy { + client Client + specification OpenAPI +} + +// creates and OpenAPI Proxy Controller +pub fn new_openapi_proxy(proxy OpenAPIProxy) OpenAPIProxy { + return proxy +} + +// creates and OpenAPI Proxy Controller +pub fn (proxy OpenAPIProxy) controller() &Controller { + // Initialize the server + mut controller := &Controller{ + specification: proxy.specification + handler: Handler{ + client: proxy.client + } + } + return controller +} + +@[params] +pub struct RunParams { +pub: + port int = 8080 +} + +fn (proxy OpenAPIProxy) run(params RunParams) { + mut controller := proxy.controller() + veb.run[Controller, Context](mut controller, params.port) +} + +pub struct Handler { +pub mut: + client Client +} + +fn (mut handler Handler) handle(request Request) !Response { + // Convert incoming OpenAPI request to a procedure call + call := rpc.openapi_request_to_procedure_call(request) + + // Process the procedure call + procedure_response := handler.client.dialogue(call, Params{ + timeout: 30 // Set timeout in seconds + }) or { + // Handle ProcedureError + if err is ProcedureError { + return Response{ + status: http.status_from_int(err.code()) // Map ProcedureError reason to HTTP status code + body: json.encode({ + 'error': err.msg() + }) + } + } + return error('Unexpected error: ${err}') + } + + // Convert returned procedure response to OpenAPI response + return Response{ + status: http.Status.ok // Assuming success if no error + body: procedure_response.result + } +} diff --git a/crystallib/hero/baobab/actor/server.v b/crystallib/hero/baobab/actor/server.v new file mode 100644 index 000000000..9ed10d8d8 --- /dev/null +++ b/crystallib/hero/baobab/actor/server.v @@ -0,0 +1,34 @@ +module actor + +import freeflowuniverse.crystallib.web.openapi { OpenAPI } +import veb + +pub struct Server { + veb.Controller +} + +pub struct Context { + veb.Context +} + +pub struct ServerConfig { + ClientConfig +pub: + openapi_spec OpenAPI +} + +pub fn new_server(cfg ServerConfig) !&Server { + mut s := &Server{} + + openapi_proxy := new_openapi_proxy( + client: new_client(cfg.ClientConfig)! + specification: cfg.openapi_spec + ) + + s.register_controller[openapi.Controller, Context]('/openapi', mut openapi_proxy.controller())! + return s +} + +pub fn (mut server Server) run(params RunParams) { + veb.run[Server, Context](mut server, params.port) +} diff --git a/crystallib/hero/baobab/example_actor/README.md b/crystallib/hero/baobab/example_actor/README.md new file mode 100644 index 000000000..4723d6967 --- /dev/null +++ b/crystallib/hero/baobab/example_actor/README.md @@ -0,0 +1,38 @@ +# Example Actor Module + +The Example Actor module serves as a reference and testable example of an actor module within the Baobab system. It showcases how actor modules are generated from specifications and can be parsed back into specifications. + +## Purpose + +This module is designed with two primary objectives: +1. Developer Reference + +Provide a concrete example of the final output of a generated actor. Developers working on Baobab can use this as a reference for: +* Understanding the structure of actor modules. +* Programming against the actor specification format. + +2. Test Code Generation + +Offer a compilable, generatable actor module to: +* Validate the functionality of Baobab’s code generation tools. +* Ensure compatibility and correctness of generated modules. + +## Features +* Demonstrates the structure of actor modules generated from specifications. +* Allows developers to see both the specification and its parsed representation. +* Can be regenerated and compiled as part of the Baobab testing process. + +## How to Use + +* Use this module as a template when developing or testing new actor specifications. +* Leverage it to verify Baobab’s code generation processes by regenerating and compiling it as needed. + +This module is integral to Baobab’s development and testing, providing both clarity and reliability for the actor generation workflow. + +# Example Actor + +The Example Actor module represents and exemplifies an actor module that is generated from specification and can be parsed. This module serves two purposes: + +1. Provide a reference on the end output of a generated actor for developers working on `baobab` to program against +2. Provide a generatable, compilable module to ensure and test code generate functionalities utilized in `baobab` + diff --git a/crystallib/hero/baobab/example_actor/actor.v b/crystallib/hero/baobab/example_actor/actor.v new file mode 100644 index 000000000..ad76d3879 --- /dev/null +++ b/crystallib/hero/baobab/example_actor/actor.v @@ -0,0 +1,36 @@ +module example_actor + +import os +import freeflowuniverse.crystallib.hero.baobab.actor {IActor, RunParams} +import freeflowuniverse.crystallib.web.openapi +import time + +const openapi_spec_path = '${os.dir(@FILE)}/specs/openapi.json' +const openapi_spec_json = os.read_file(openapi_spec_path) or { panic(err) } +const openapi_specification = openapi.json_decode(openapi_spec_json)! + +struct ExampleActor { + actor.Actor +} + +fn new() !ExampleActor { + return ExampleActor{ + actor.new('example') + } +} + +pub fn run() ! { + mut a_ := new()! + mut a := IActor(a_) + a.run()! +} + +pub fn run_server(params RunParams) ! { + mut a := new()! + mut server := actor.new_server( + redis_url: 'localhost:6379' + redis_queue: a.name + openapi_spec: openapi_specification + )! + server.run(params) +} \ No newline at end of file diff --git a/crystallib/hero/baobab/example_actor/actor_test.v b/crystallib/hero/baobab/example_actor/actor_test.v new file mode 100644 index 000000000..cab2e268f --- /dev/null +++ b/crystallib/hero/baobab/example_actor/actor_test.v @@ -0,0 +1,17 @@ +module example_actor + +const test_port = 8101 + +pub fn test_new() ! { + new() or { + return error('Failed to create actor:\n${err}') + } +} + +pub fn test_run() ! { + spawn run() +} + +pub fn test_run_server() ! { + spawn run_server(port: test_port) +} \ No newline at end of file diff --git a/crystallib/hero/baobab/example_actor/handle.v b/crystallib/hero/baobab/example_actor/handle.v new file mode 100644 index 000000000..dfc067ef4 --- /dev/null +++ b/crystallib/hero/baobab/example_actor/handle.v @@ -0,0 +1,5 @@ +module example_actor + +pub fn (mut a ExampleActor) handle(method string, data string) !string { + return data +} \ No newline at end of file diff --git a/crystallib/hero/baobab/example_actor/interface_command.v b/crystallib/hero/baobab/example_actor/interface_command.v new file mode 100644 index 000000000..c063ab088 --- /dev/null +++ b/crystallib/hero/baobab/example_actor/interface_command.v @@ -0,0 +1,83 @@ +module publishing + +import freeflowuniverse.crystallib.core.pathlib +import cli { Command, Flag } +import os +import freeflowuniverse.crystallib.ui.console + +pub fn cmd_example_actor() Command { + mut cmd := Command{ + name: 'example_actor' + usage: '' + description: 'create, edit, show mdbooks' + required_args: 0 + execute: cmd_example_actor_execute + } + + mut cmd_list := Command{ + sort_flags: true + name: 'list_books' + execute: cmd_publisher_list_books + description: 'will list existing mdbooks' + pre_execute: pre_func + } + + mut cmd_open := Command{ + name: 'open' + execute: cmd_publisher_open + description: 'will open the publication with the provided name' + pre_execute: pre_func + } + + cmd_open.add_flag(Flag{ + flag: .string + name: 'name' + abbrev: 'n' + description: 'name of the publication.' + }) + + cmd.add_command(cmd_list) + cmd.add_command(cmd_open) + return cmd +} + +fn cmd_publisher_list_books(cmd Command) ! { + console.print_header('Books:') + books := publisher.list_books()! + for book in books { + console.print_stdout(book.str()) + } +} + +fn cmd_publisher_open(cmd Command) ! { + name := cmd.flags.get_string('name') or { '' } + publisher.open(name)! +} + +fn cmd_execute(cmd Command) ! { + mut name := cmd.flags.get_string('name') or { '' } + + if name == '' { + console.print_debug('did not find name of book to generate, check in heroscript or specify with --name') + publisher_help(cmd) + exit(1) + } + + edit := cmd.flags.get_bool('edit') or { false } + open := cmd.flags.get_bool('open') or { false } + if edit || open { + // mdbook.book_open(name)! + } + + if edit { + // publisher.book_edit(name)! + } +} + +fn publisher_help(cmd Command) { + console.clear() + console.print_header('Instructions for example actor:') + console.print_lf(1) + console.print_stdout(cmd.help_message()) + console.print_lf(5) +} \ No newline at end of file diff --git a/crystallib/hero/baobab/example_actor/scripts/compile.sh b/crystallib/hero/baobab/example_actor/scripts/compile.sh new file mode 100644 index 000000000..e69de29bb diff --git a/crystallib/hero/baobab/example_actor/scripts/run.sh b/crystallib/hero/baobab/example_actor/scripts/run.sh new file mode 100644 index 000000000..e69de29bb diff --git a/crystallib/baobab/generator/README.md b/crystallib/hero/baobab/generator/README.md similarity index 65% rename from crystallib/baobab/generator/README.md rename to crystallib/hero/baobab/generator/README.md index 6dc2eab32..0e3e45534 100644 --- a/crystallib/baobab/generator/README.md +++ b/crystallib/hero/baobab/generator/README.md @@ -1,4 +1,22 @@ -# ModelGenerator +# Generator + +The Generator synchronizes actor code and specifications, allowing bidirectional transformation between the two. + +This a + + + +## Development Workflow + +A sample development workflow using the generator would be like: +1. generating actor specification from an actor openrpc / openapi specification (see [specification reflection](specification/#reflection)) +2. generating actor code from the actor specification +3. updating actor code by filling in method prototypes +4. adding methods to the actor to develop actor further +5. parsing specification back from actor + +6. regenerating actor from the specification +this allows for - a tool which takes dir as input - is just some v files which define models diff --git a/crystallib/hero/baobab/generator/generate_actor.v b/crystallib/hero/baobab/generator/generate_actor.v new file mode 100644 index 000000000..d53a110e1 --- /dev/null +++ b/crystallib/hero/baobab/generator/generate_actor.v @@ -0,0 +1,111 @@ +module generator + +import freeflowuniverse.crystallib.core.codemodel { Folder, IFile, VFile, CodeItem, File, Function, Import, Module, Struct, CustomCode } +import freeflowuniverse.crystallib.core.texttools +import freeflowuniverse.crystallib.core.codeparser +import freeflowuniverse.crystallib.data.markdownparser +import freeflowuniverse.crystallib.data.markdownparser.elements { Header } +import freeflowuniverse.crystallib.rpc.openrpc +import freeflowuniverse.crystallib.core.pathlib +import freeflowuniverse.crystallib.hero.baobab.specification {ActorMethod, ActorSpecification, ActorInterface} +import os +import json + +@[params] +pub struct Params { +pub: + interfaces []ActorInterface // the interfaces to be supported +} + +pub fn generate_actor_module(spec ActorSpecification, params Params) !Module { + mut files := []IFile{} + + files = [ + generate_readme_file(spec)!, + generate_actor_file(spec)!, + generate_actor_test_file(spec)!, + generate_handle_file(spec)!, + generate_methods_file(spec)! + generate_client_file(spec)! + ] + + mut docs_files := []IFile{} + + // generate code files for supported interfaces + for iface in params.interfaces { + match iface { + .openrpc { + // convert actor spec to openrpc spec + openrpc_spec := spec.to_openrpc() + + // generate openrpc code files + files << generate_openrpc_client_file(openrpc_spec)! + files << generate_openrpc_client_test_file(openrpc_spec)! + + // add openrpc.json to docs + docs_files << generate_openrpc_file(openrpc_spec)! + } + .command { + files << generate_command_file(spec)! + } + else { + return error('unsupported interface ${iface}') + } + } + } + + // folder with docs + docs_folder := Folder { + name: 'docs' + files: docs_files + } + + // create module with code files and docs folder + name_fixed := texttools.name_fix_snake(spec.name) + return codemodel.new_module( + name: '${name_fixed}_actor' + files: files + folders: [docs_folder] + ) +} + +fn generate_readme_file(spec ActorSpecification) !File { + return File{ + name: 'README' + extension: 'md' + content: '# ${spec.name}\n${spec.description}' + } +} + +fn generate_actor_file(spec ActorSpecification) !VFile { + dollar := '$' + actor_name_snake := texttools.name_fix_snake(spec.name) + actor_name_pascal := texttools.name_fix_snake_to_pascal(spec.name) + code := $tmpl('./templates/actor.v.template') + return VFile { + name: 'actor' + items: [CustomCode{code}] + } +} + +fn generate_actor_test_file(spec ActorSpecification) !VFile { + dollar := '$' + actor_name_snake := texttools.name_fix_snake(spec.name) + actor_name_pascal := texttools.name_fix_snake_to_pascal(spec.name) + code := $tmpl('./templates/actor_test.v.template') + return VFile { + name: 'actor_test' + items: [CustomCode{code}] + } +} + + +pub fn generate_openapi_file(spec ActorSpecification) !File { + openapi_spec := spec.to_openapi() + openapi_json := json.encode(openapi_spec) + return File{ + name: 'openapi' + extension: 'json' + content: openapi_json + } +} \ No newline at end of file diff --git a/crystallib/hero/baobab/generator/generate_actor_test.v b/crystallib/hero/baobab/generator/generate_actor_test.v new file mode 100644 index 000000000..7163ff094 --- /dev/null +++ b/crystallib/hero/baobab/generator/generate_actor_test.v @@ -0,0 +1,136 @@ +module generator + +import freeflowuniverse.crystallib.core.codemodel +import freeflowuniverse.crystallib.hero.baobab.specification +import freeflowuniverse.crystallib.core.codeparser +import freeflowuniverse.crystallib.core.pathlib +import os + +const actor_spec = specification.ActorSpecification{ + name: 'Pet Store' + description: 'A sample API for a pet store' + interfaces: [.openrpc, .command] + methods: [specification.ActorMethod{ + name: 'listPets' + description: 'List all pets' + func: codemodel.Function{ + name: 'listPets' + params: [codemodel.Param{ + description: 'Maximum number of pets to return' + name: 'limit' + typ: codemodel.Type{ + symbol: 'int' + } + }] + } + }, specification.ActorMethod{ + name: 'createPet' + description: 'Create a new pet' + func: codemodel.Function{ + name: 'createPet' + } + }, specification.ActorMethod{ + name: 'getPet' + description: 'Get a pet by ID' + func: codemodel.Function{ + name: 'getPet' + params: [codemodel.Param{ + required: true + description: 'ID of the pet to retrieve' + name: 'petId' + typ: codemodel.Type{ + symbol: 'int' + } + }] + } + }, specification.ActorMethod{ + name: 'deletePet' + description: 'Delete a pet by ID' + func: codemodel.Function{ + name: 'deletePet' + params: [codemodel.Param{ + required: true + description: 'ID of the pet to delete' + name: 'petId' + typ: codemodel.Type{ + symbol: 'int' + } + }] + } + }, specification.ActorMethod{ + name: 'listOrders' + description: 'List all orders' + func: codemodel.Function{ + name: 'listOrders' + } + }, specification.ActorMethod{ + name: 'getOrder' + description: 'Get an order by ID' + func: codemodel.Function{ + name: 'getOrder' + params: [codemodel.Param{ + required: true + description: 'ID of the order to retrieve' + name: 'orderId' + typ: codemodel.Type{ + symbol: 'int' + } + }] + } + }, specification.ActorMethod{ + name: 'deleteOrder' + description: 'Delete an order by ID' + func: codemodel.Function{ + name: 'deleteOrder' + params: [codemodel.Param{ + required: true + description: 'ID of the order to delete' + name: 'orderId' + typ: codemodel.Type{ + symbol: 'int' + } + }] + } + }, specification.ActorMethod{ + name: 'createUser' + description: 'Create a user' + func: codemodel.Function{ + name: 'createUser' + } + }] + objects: [specification.BaseObject{ + structure: codemodel.Struct{ + name: 'Pet' + } + }, specification.BaseObject{ + structure: codemodel.Struct{ + name: 'NewPet' + } + }, specification.BaseObject{ + structure: codemodel.Struct{ + name: 'Pets' + } + }, specification.BaseObject{ + structure: codemodel.Struct{ + name: 'Order' + } + }, specification.BaseObject{ + structure: codemodel.Struct{ + name: 'User' + } + }, specification.BaseObject{ + structure: codemodel.Struct{ + name: 'NewUser' + } + }] +} + +const destination = '${os.dir(@FILE)}/testdata' + +fn test_generate_actor_module() { + actor_module := generate_actor_module(actor_spec)! + actor_module.write(destination, + format: true + overwrite: true + )! +} diff --git a/crystallib/hero/baobab/generator/generate_clients.v b/crystallib/hero/baobab/generator/generate_clients.v new file mode 100644 index 000000000..f555b49ae --- /dev/null +++ b/crystallib/hero/baobab/generator/generate_clients.v @@ -0,0 +1,65 @@ +module generator + +import freeflowuniverse.crystallib.core.codemodel { Folder, IFile, VFile, CodeItem, File, Function, Import, Module, Struct, CustomCode } +import freeflowuniverse.crystallib.core.texttools +import freeflowuniverse.crystallib.core.codeparser +import freeflowuniverse.crystallib.data.markdownparser +import freeflowuniverse.crystallib.data.markdownparser.elements { Header } +import freeflowuniverse.crystallib.rpc.openrpc +import freeflowuniverse.crystallib.core.pathlib +import freeflowuniverse.crystallib.hero.baobab.specification {ActorMethod, ActorSpecification} +import os +import json + +pub fn generate_client_file(spec ActorSpecification) !VFile { + actor_name_snake := texttools.name_fix_snake(spec.name) + actor_name_pascal := texttools.name_fix_snake_to_pascal(spec.name) + + mut items := []CodeItem{} + + items << CustomCode {' + pub struct Client { + actor.Client + } + + fn new_client() Client { + return Client{} + }'} + + for method in spec.methods { + items << CustomCode{generate_client_method(method)!} + } + + return VFile { + imports: [ + Import{ + mod: 'freeflowuniverse.crystallib.data.paramsparser' + }, + Import{ + mod: 'freeflowuniverse.crystallib.hero.baobab.actor' + } + ] + name: 'client' + items: items + } +} + +pub fn generate_client_method(method ActorMethod) !string { + name_fixed := texttools.name_fix_snake(method.name) + mut handler := '// Method for ${name_fixed}\n' + params := if method.func.params.len > 0 { + method.func.params.map(it.vgen()).join(', ') + } else {''} + + call_params := if method.func.params.len > 0 { + method.func.params.map(it.name).join(', ') + } else {''} + + handler += "fn (mut client Client) ${name_fixed}(${params}) ! { + client.call_to_action( + method: ${name_fixed} + params: paramsparser.encode(${call_params}) + ) + }" + return handler +} \ No newline at end of file diff --git a/crystallib/hero/baobab/generator/generate_command.v b/crystallib/hero/baobab/generator/generate_command.v new file mode 100644 index 000000000..a4211d602 --- /dev/null +++ b/crystallib/hero/baobab/generator/generate_command.v @@ -0,0 +1,79 @@ +module generator + +import freeflowuniverse.crystallib.core.codemodel { Folder, IFile, VFile, CodeItem, File, Function, Import, Module, Struct, CustomCode } +import freeflowuniverse.crystallib.core.texttools +import freeflowuniverse.crystallib.hero.baobab.specification {ActorMethod, ActorSpecification} + +pub fn generate_command_file(spec ActorSpecification) !VFile { + mut items := []CodeItem{} + items << CustomCode{generate_cmd_function(spec)} + for i in spec.methods { + items << CustomCode{generate_method_cmd_function(spec.name, i)} + } + return VFile { + name: 'command' + imports: [ + Import{ + mod: 'freeflowuniverse.crystallib.ui.console' + }, + Import{ + mod: 'cli' + types: ['Command', 'Flag'] + } + ] + items: items + } +} + +pub fn generate_cmd_function(spec ActorSpecification) string { + actor_name_snake := texttools.name_fix_snake(spec.name) + mut cmd_function := " + pub fn cmd() Command { + mut cmd := Command{ + name: '${actor_name_snake}' + usage: '' + description: '${spec.description}' + execute: cmd_execute + } + " + + mut method_cmds := []string{} + for method in spec.methods { + method_cmds << generate_method_cmd(method) + } + + cmd_function += '${method_cmds.join_lines()}}' + + return cmd_function +} + +pub fn generate_method_cmd(method ActorMethod) string { + method_name_snake := texttools.name_fix_snake(method.name) + return " + mut cmd_${method_name_snake} := Command{ + sort_flags: true + name: '${method_name_snake}' + execute: cmd_${method_name_snake}_execute + description: '${method.description}' + } + " +} + +pub fn generate_method_cmd_function(actor_name string, method ActorMethod) string { + mut operation_handlers := []string{} + mut routes := []string{} + + actor_name_snake := texttools.name_fix_snake(actor_name) + method_name_snake := texttools.name_fix_snake(method.name) + + method_call := if method.func.result.typ.symbol == '' { + '${actor_name_snake}.${method_name_snake}()!' + } else { + 'result := ${actor_name_snake}.${method_name_snake}()!' + } + return ' + fn cmd_${method_name_snake}(cmd Command) ! { + ${method_call} + } + ' +} diff --git a/crystallib/hero/baobab/generator/generate_handle.v b/crystallib/hero/baobab/generator/generate_handle.v new file mode 100644 index 000000000..5187db2e7 --- /dev/null +++ b/crystallib/hero/baobab/generator/generate_handle.v @@ -0,0 +1,82 @@ +module generator + +import freeflowuniverse.crystallib.core.codemodel { Folder, IFile, VFile, CodeItem, File, Function, Import, Module, Struct, CustomCode } +import freeflowuniverse.crystallib.core.texttools +import freeflowuniverse.crystallib.core.codeparser +import freeflowuniverse.crystallib.data.markdownparser +import freeflowuniverse.crystallib.data.markdownparser.elements { Header } +import freeflowuniverse.crystallib.rpc.openrpc +import freeflowuniverse.crystallib.core.pathlib +import freeflowuniverse.crystallib.hero.baobab.specification {ActorMethod, ActorSpecification} +import os +import json + +fn generate_handle_file(spec ActorSpecification) !VFile { + mut items := []CodeItem{} + items << CustomCode{generate_handle_function(spec)} + for method in spec.methods { + items << CustomCode{generate_method_handle(spec.name, method)!} + } + return VFile { + name: 'act' + items: items + } +} + +pub fn generate_handle_function(spec ActorSpecification) string { + actor_name_pascal := texttools.name_fix_snake_to_pascal(spec.name) + mut operation_handlers := []string{} + mut routes := []string{} + + // Iterate over OpenAPI paths and operations + for method in spec.methods { + operation_id := method.name + params := method.func.params.map(it.name).join(', ') + + // Generate route case + route := generate_route_case(method.name, operation_id) + routes << route + } + + // Combine the generated handlers and main router into a single file + return [ + '// AUTO-GENERATED FILE - DO NOT EDIT MANUALLY', + '', + 'pub fn (mut actor ${actor_name_pascal}Actor) act(action Action) !Response {', + ' match action.name {', + routes.join('\n'), + ' else {', + ' return error("Unknown operation: \${req.operation.operation_id}")', + ' }', + ' }', + '}', + ].join('\n') +} + +pub fn generate_method_handle(actor_name string, method ActorMethod) !string { + actor_name_pascal := texttools.name_fix_snake_to_pascal(actor_name) + name_fixed := texttools.name_fix_snake(method.name) + mut handler := '// Handler for ${name_fixed}\n' + handler += "fn (mut actor ${actor_name_pascal}Actor) handle_${name_fixed}(data string) !string {\n" + if method.func.params.len > 0 { + handler += ' params := json.decode(${method.func.params[0].typ.symbol}, data) or { return error("Invalid input data: \${err}") }\n' + handler += ' result := actor.${name_fixed}(params)\n' + } else { + handler += ' result := actor.${name_fixed}()\n' + } + handler += ' return json.encode(result)\n' + handler += '}' + return handler +} + +// Helper function to generate a case block for the main router +fn generate_route_case(method string, operation_id string) string { + name_fixed := texttools.name_fix_snake(operation_id) + mut case_block := ' "${operation_id}" {' + case_block += '\n response := actor.handle_${name_fixed}(req.body) or {' + case_block += '\n return Response{ status: http.Status.internal_server_error, body: "Internal server error: \${err}" }' + case_block += '\n }' + case_block += '\n return Response{ status: http.Status.ok, body: response }' + case_block += '\n }' + return case_block +} \ No newline at end of file diff --git a/crystallib/hero/baobab/generator/generate_methods.v b/crystallib/hero/baobab/generator/generate_methods.v new file mode 100644 index 000000000..a7296992c --- /dev/null +++ b/crystallib/hero/baobab/generator/generate_methods.v @@ -0,0 +1,38 @@ +module generator + +import freeflowuniverse.crystallib.core.codemodel { Folder, IFile, VFile, CodeItem, File, Function, Import, Module, Struct, CustomCode } +import freeflowuniverse.crystallib.core.texttools +import freeflowuniverse.crystallib.core.codeparser +import freeflowuniverse.crystallib.data.markdownparser +import freeflowuniverse.crystallib.data.markdownparser.elements { Header } +import freeflowuniverse.crystallib.rpc.openrpc +import freeflowuniverse.crystallib.core.pathlib +import freeflowuniverse.crystallib.hero.baobab.specification {ActorMethod, ActorSpecification} +import os +import json + +pub fn generate_methods_file(spec ActorSpecification) !VFile { + actor_name_snake := texttools.name_fix_snake(spec.name) + actor_name_pascal := texttools.name_fix_snake_to_pascal(spec.name) + + mut items := []CodeItem{} + for method in spec.methods { + items << CustomCode{generate_method_function(spec.name, method)!} + } + + return VFile { + name: 'methods' + items: items + } +} + +pub fn generate_method_function(actor_name string, method ActorMethod) !string { + actor_name_pascal := texttools.name_fix_snake_to_pascal(actor_name) + name_fixed := texttools.name_fix_snake(method.name) + mut handler := '// Method for ${name_fixed}\n' + params := if method.func.params.len > 0 { + method.func.params.map(it.vgen()).join(', ') + } else {''} + handler += "fn (mut actor ${actor_name_pascal}Actor) ${name_fixed}(${params}) ! {}" + return handler +} \ No newline at end of file diff --git a/crystallib/hero/baobab/generator/generate_objects.v b/crystallib/hero/baobab/generator/generate_objects.v new file mode 100644 index 000000000..c9d27d704 --- /dev/null +++ b/crystallib/hero/baobab/generator/generate_objects.v @@ -0,0 +1,48 @@ +module generator + +// pub fn generate_object_code(actor Struct, object BaseObject) VFile { +// obj_name := texttools.name_fix_snake(object.structure.name) +// object_type := object.structure.name + +// mut items := []CodeItem{} +// items = [generate_new_method(actor, object), generate_get_method(actor, object), +// generate_set_method(actor, object), generate_delete_method(actor, object), +// generate_list_result_struct(actor, object), generate_list_method(actor, object)] + +// items << generate_object_methods(actor, object) +// mut file := codemodel.new_file( +// mod: texttools.name_fix(actor.name) +// name: obj_name +// imports: [ +// Import{ +// mod: object.structure.mod +// types: [object_type] +// }, +// Import{ +// mod: 'freeflowuniverse.crystallib.baobab.backend' +// types: ['FilterParams'] +// }, +// ] +// items: items +// ) + +// if object.structure.fields.any(it.attrs.any(it.name == 'index')) { +// // can't filter without indices +// filter_params := generate_filter_params(actor, object) +// file.items << filter_params.map(CodeItem(it)) +// file.items << generate_filter_method(actor, object) +// } + +// return file +// } + + +// pub fn (a Actor) generate_model_files() ![]VFile { +// structs := a.objects.map(it.structure) +// return a.objects.map(codemodel.new_file( +// mod: texttools.name_fix(a.name) +// name: '${texttools.name_fix(it.structure.name)}_model' +// // imports: [Import{mod:'freeflowuniverse.crystallib.baobab.actor'}] +// items: [it.structure] +// )) +// } diff --git a/crystallib/hero/baobab/generator/generate_openrpc.v b/crystallib/hero/baobab/generator/generate_openrpc.v new file mode 100644 index 000000000..05a4e7347 --- /dev/null +++ b/crystallib/hero/baobab/generator/generate_openrpc.v @@ -0,0 +1,113 @@ +module generator + +import json +import freeflowuniverse.crystallib.core.codemodel { VFile, File, Function, Module, Struct } +import freeflowuniverse.crystallib.core.pathlib +import freeflowuniverse.crystallib.core.texttools +import freeflowuniverse.crystallib.hero.baobab.specification {ActorSpecification} +import freeflowuniverse.crystallib.rpc.openrpc { Components, OpenRPC } +import freeflowuniverse.crystallib.data.jsonschema { SchemaRef } + +pub fn generate_openrpc_file(spec OpenRPC) !File { + return File { + name: 'openrpc' + extension: 'json' + content: json.encode(spec) + } +} + +pub fn generate_openrpc_client_file(spec OpenRPC) !VFile { + mut objects_map := map[string]Struct{} + // for object in spec.objects { + // objects_map[object.structure.name] = object.structure + // } + client_file := spec.generate_client_file(objects_map)! + return VFile { + ...client_file, + name: 'client_openrpc' + } +} + +pub fn generate_openrpc_client_test_file(spec OpenRPC) !VFile { + mut objects_map := map[string]Struct{} + // for object in spec.objects { + // objects_map[object.structure.name] = object.structure + // } + mut methods_map := map[string]Function{} + // for method in spec.methods { + // methods_map[method.func.name] = method.func + // } + file := spec.generate_client_test_file(methods_map, objects_map)! + return VFile { + ...file, + name: 'client_openrpc_test' + } +} + +// pub fn (actor Actor) generate_openrpc_code() !Module { +// openrpc_obj := actor.generate_openrpc() +// openrpc_json := openrpc_obj.encode()! + +// openrpc_file := File{ +// name: 'openrpc' +// extension: 'json' +// content: openrpc_json +// } + +// mut methods_map := map[string]Function{} +// for method in actor.methods { +// methods_map[method.func.name] = method.func +// } + +// mut objects_map := map[string]Struct{} +// for object in actor.objects { +// objects_map[object.structure.name] = object.structure +// } +// // actor_struct := generate_actor_struct(actor.name) +// actor_struct := actor.structure + +// client_file := openrpc_obj.generate_client_file(objects_map)! +// client_test_file := openrpc_obj.generate_client_test_file(methods_map, objects_map)! + +// handler_file := openrpc_obj.generate_handler_file(actor_struct, methods_map, objects_map)! +// handler_test_file := openrpc_obj.generate_handler_test_file(actor_struct, methods_map, +// objects_map)! + +// server_file := openrpc_obj.generate_server_file()! +// server_test_file := openrpc_obj.generate_server_test_file()! + +// return Module{ +// files: [ +// client_file, +// client_test_file, +// handler_file, +// handler_test_file, +// server_file, +// server_test_file, +// ] +// // misc_files: [openrpc_file] +// } +// } + +// pub fn (mut a Actor) export_playground(path string, openrpc_path string) ! { +// dollar := '$' +// openrpc.export_playground( +// dest: pathlib.get_dir(path: '${path}/playground')! +// specs: [ +// pathlib.get(openrpc_path), +// ] +// )! +// mut cli_file := pathlib.get_file(path: '${path}/command/cli.v')! +// cli_file.write($tmpl('./templates/playground.v.template'))! +// } + +// pub fn param_to_content_descriptor(param Param) openrpc.ContentDescriptor { +// if param.name == 'id' && param.typ.symbol == + +// return openrpc.ContentDescriptor { +// name: param.name +// summary: param.description +// required: param.is_required() +// schema: +// } +// } diff --git a/crystallib/hero/baobab/generator/generate_openrpc_test.v b/crystallib/hero/baobab/generator/generate_openrpc_test.v new file mode 100644 index 000000000..5f5800ca6 --- /dev/null +++ b/crystallib/hero/baobab/generator/generate_openrpc_test.v @@ -0,0 +1,40 @@ +module generator + +import freeflowuniverse.crystallib.core.codemodel { Function, Param, Result, Struct, Type } +import freeflowuniverse.crystallib.rpc.openrpc + +const test_actor_specification = ActorSpecification { + methods: [ + ActorMethod{ + func: Function{ + name: 'get_object' + params: [ + Param{ + name: 'id' + typ: Type{ + symbol: 'int' + } + }, + ] + result: Result{ + typ: Type{ + symbol: 'Object' + } + } + } + }, + ] + objects: [BaseObject{ + structure: Struct{ + name: 'Object' + } + }] +} + +pub fn test_generate_openrpc() ! { + actor := Actor{ + + } + object := generate_openrpc(actor) + panic(object.encode()!) +} diff --git a/crystallib/hero/baobab/generator/templates/actor.v.template b/crystallib/hero/baobab/generator/templates/actor.v.template new file mode 100644 index 000000000..0e1c5b2ae --- /dev/null +++ b/crystallib/hero/baobab/generator/templates/actor.v.template @@ -0,0 +1,34 @@ +import os +import freeflowuniverse.crystallib.hero.baobab.actor {IActor, RunParams} +import freeflowuniverse.crystallib.web.openapi +import time + +const openapi_spec_path = '@{dollar}{os.dir(@@FILE)}/specs/openapi.json' +const openapi_spec_json = os.read_file(openapi_spec_path) or { panic(err) } +const openapi_specification = openapi.json_decode(openapi_spec_json)! + +struct @{actor_name_pascal}Actor { + actor.Actor +} + +fn new() !@{actor_name_pascal}Actor { + return @{actor_name_pascal}Actor { + actor.new('@{actor_name_snake}') + } +} + +pub fn run() ! { + mut a_ := new()! + mut a := IActor(a_) + a.run()! +} + +pub fn run_server(params RunParams) ! { + mut a := new()! + mut server := actor.new_server( + redis_url: 'localhost:6379' + redis_queue: a.name + openapi_spec: openapi_specification + )! + server.run(params) +} \ No newline at end of file diff --git a/crystallib/hero/baobab/generator/templates/actor_test.v.template b/crystallib/hero/baobab/generator/templates/actor_test.v.template new file mode 100644 index 000000000..899471fea --- /dev/null +++ b/crystallib/hero/baobab/generator/templates/actor_test.v.template @@ -0,0 +1,15 @@ +const test_port = 8101 + +pub fn test_new() ! { + new() or { + return error('Failed to create actor:\n@{dollar}{err}') + } +} + +pub fn test_run() ! { + spawn run() +} + +pub fn test_run_server() ! { + spawn run_server(port: test_port) +} \ No newline at end of file diff --git a/crystallib/baobab/generator/templates/cli.v.template b/crystallib/hero/baobab/generator/templates/cli.v.template similarity index 100% rename from crystallib/baobab/generator/templates/cli.v.template rename to crystallib/hero/baobab/generator/templates/cli.v.template diff --git a/crystallib/hero/baobab/generator/templates/command.v.template b/crystallib/hero/baobab/generator/templates/command.v.template new file mode 100644 index 000000000..8412b970a --- /dev/null +++ b/crystallib/hero/baobab/generator/templates/command.v.template @@ -0,0 +1,81 @@ +import freeflowuniverse.crystallib.core.pathlib +import cli { Command, Flag } +import os +import freeflowuniverse.crystallib.ui.console + +pub fn cmd_example_actor() Command { + mut cmd := Command{ + name: 'example_actor' + usage: '' + description: 'create, edit, show mdbooks' + required_args: 0 + execute: cmd_example_actor_execute + } + + mut cmd_list := Command{ + sort_flags: true + name: 'list_books' + execute: cmd_publisher_list_books + description: 'will list existing mdbooks' + pre_execute: pre_func + } + + mut cmd_open := Command{ + name: 'open' + execute: cmd_publisher_open + description: 'will open the publication with the provided name' + pre_execute: pre_func + } + + cmd_open.add_flag(Flag{ + flag: .string + name: 'name' + abbrev: 'n' + description: 'name of the publication.' + }) + + cmd.add_command(cmd_list) + cmd.add_command(cmd_open) + return cmd +} + +fn cmd_publisher_list_books(cmd Command) ! { + console.print_header('Books:') + books := publisher.list_books()! + for book in books { + console.print_stdout(book.str()) + } +} + +fn cmd_publisher_open(cmd Command) ! { + name := cmd.flags.get_string('name') or { '' } + publisher.open(name)! +} + +fn cmd_execute(cmd Command) ! { + mut name := cmd.flags.get_string('name') or { '' } + + if name == '' { + console.print_debug('did not find name of book to generate, check in heroscript or specify with --name') + publisher_help(cmd) + exit(1) + } + + edit := cmd.flags.get_bool('edit') or { false } + open := cmd.flags.get_bool('open') or { false } + if edit || open { + // mdbook.book_open(name)! + } + + if edit { + // publisher.book_edit(name)! + } +} + +fn publisher_help(cmd Command) { + console.clear() + console.print_header('Instructions for example actor:') + console.print_lf(1) + console.print_stdout(cmd.help_message()) + console.print_lf(5) +} \ No newline at end of file diff --git a/crystallib/baobab/generator/templates/playground.v.template b/crystallib/hero/baobab/generator/templates/playground.v.template similarity index 100% rename from crystallib/baobab/generator/templates/playground.v.template rename to crystallib/hero/baobab/generator/templates/playground.v.template diff --git a/crystallib/hero/baobab/generator/testdata/.gitignore b/crystallib/hero/baobab/generator/testdata/.gitignore new file mode 100644 index 000000000..fbe24508f --- /dev/null +++ b/crystallib/hero/baobab/generator/testdata/.gitignore @@ -0,0 +1 @@ +pet_store_actor \ No newline at end of file diff --git a/crystallib/hero/baobab/generator/testdata/pet_store_actor/README.md b/crystallib/hero/baobab/generator/testdata/pet_store_actor/README.md new file mode 100644 index 000000000..b7813ca56 --- /dev/null +++ b/crystallib/hero/baobab/generator/testdata/pet_store_actor/README.md @@ -0,0 +1,2 @@ +# Pet Store +A sample API for a pet store \ No newline at end of file diff --git a/crystallib/hero/baobab/generator/testdata/pet_store_actor/actor.v b/crystallib/hero/baobab/generator/testdata/pet_store_actor/actor.v new file mode 100644 index 000000000..21e9cda91 --- /dev/null +++ b/crystallib/hero/baobab/generator/testdata/pet_store_actor/actor.v @@ -0,0 +1,34 @@ +module pet_store_actor + +import os +import freeflowuniverse.crystallib.hero.baobab.actor { IActor, RunParams } +import freeflowuniverse.crystallib.web.openapi +import time + +const openapi_spec_path = '${os.dir(@FILE)}/specs/openapi.json' +const openapi_spec_json = os.read_file(openapi_spec_path) or { panic(err) } +const openapi_specification = openapi.json_decode(openapi_spec_json)! + +struct PetStoreActor { + actor.Actor +} + +fn new() !PetStoreActor { + return PetStoreActor{actor.new('pet_store')} +} + +pub fn run() ! { + mut a_ := new()! + mut a := IActor(a_) + a.run()! +} + +pub fn run_server(params RunParams) ! { + mut a := new()! + mut server := actor.new_server( + redis_url: 'localhost:6379' + redis_queue: a.name + openapi_spec: openapi_specification + )! + server.run(params) +} diff --git a/crystallib/hero/baobab/generator/testdata/pet_store_actor/actor_test.v b/crystallib/hero/baobab/generator/testdata/pet_store_actor/actor_test.v new file mode 100644 index 000000000..f8c8622c6 --- /dev/null +++ b/crystallib/hero/baobab/generator/testdata/pet_store_actor/actor_test.v @@ -0,0 +1,15 @@ +module pet_store_actor + +const test_port = 8101 + +pub fn test_new() ! { + new() or { return error('Failed to create actor:\n${err}') } +} + +pub fn test_run() ! { + spawn run() +} + +pub fn test_run_server() ! { + spawn run_server(port: test_port) +} diff --git a/crystallib/hero/baobab/generator/testdata/pet_store_actor/client_openrpc.v b/crystallib/hero/baobab/generator/testdata/pet_store_actor/client_openrpc.v new file mode 100644 index 000000000..bf13ab5ca --- /dev/null +++ b/crystallib/hero/baobab/generator/testdata/pet_store_actor/client_openrpc.v @@ -0,0 +1,27 @@ +module pet_store_actor + +import freeflowuniverse.crystallib.rpc.jsonrpc +import freeflowuniverse.crystallib.rpc.rpcwebsocket +import log + +struct Client { +mut: + transport jsonrpc.IRpcTransportClient +} + +@[params] +pub struct WsClientConfig { + address string + logger log.Logger +} + +pub fn new_ws_client(config WsClientConfig) !&Client { + mut transport := rpcwebsocket.new_rpcwsclient(config.address, config.logger) or { + return error('Failed to create RPC Websocket Client\n${err}') + } + spawn transport.run() + c := Client{ + transport: transport + } + return &c +} diff --git a/crystallib/hero/baobab/generator/testdata/pet_store_actor/client_openrpc_test.v b/crystallib/hero/baobab/generator/testdata/pet_store_actor/client_openrpc_test.v new file mode 100644 index 000000000..e2300787b --- /dev/null +++ b/crystallib/hero/baobab/generator/testdata/pet_store_actor/client_openrpc_test.v @@ -0,0 +1,11 @@ +module pet_store_actor + +import freeflowuniverse.crystallib.rpc.jsonrpc +import freeflowuniverse.crystallib.rpc.rpcwebsocket +import log + +const port = 3100 + +pub fn test_new_ws_client() ! { + mut client := new_ws_client(address: 'ws://127.0.0.1:${port}')! +} diff --git a/crystallib/hero/baobab/generator/testdata/pet_store_actor/command.v b/crystallib/hero/baobab/generator/testdata/pet_store_actor/command.v new file mode 100644 index 000000000..010c94998 --- /dev/null +++ b/crystallib/hero/baobab/generator/testdata/pet_store_actor/command.v @@ -0,0 +1,101 @@ +module pet_store_actor + +import freeflowuniverse.crystallib.ui.console +import cli { Command } + +pub fn cmd() Command { + mut cmd := Command{ + name: 'pet_store' + usage: '' + description: 'A sample API for a pet store' + execute: cmd_execute + } + + mut cmd_list_pets := Command{ + sort_flags: true + name: 'list_pets' + execute: cmd_list_pets_execute + description: 'List all pets' + } + + mut cmd_create_pet := Command{ + sort_flags: true + name: 'create_pet' + execute: cmd_create_pet_execute + description: 'Create a new pet' + } + + mut cmd_get_pet := Command{ + sort_flags: true + name: 'get_pet' + execute: cmd_get_pet_execute + description: 'Get a pet by ID' + } + + mut cmd_delete_pet := Command{ + sort_flags: true + name: 'delete_pet' + execute: cmd_delete_pet_execute + description: 'Delete a pet by ID' + } + + mut cmd_list_orders := Command{ + sort_flags: true + name: 'list_orders' + execute: cmd_list_orders_execute + description: 'List all orders' + } + + mut cmd_get_order := Command{ + sort_flags: true + name: 'get_order' + execute: cmd_get_order_execute + description: 'Get an order by ID' + } + + mut cmd_delete_order := Command{ + sort_flags: true + name: 'delete_order' + execute: cmd_delete_order_execute + description: 'Delete an order by ID' + } + + mut cmd_create_user := Command{ + sort_flags: true + name: 'create_user' + execute: cmd_create_user_execute + description: 'Create a user' + } +} + +fn cmd_list_pets(cmd Command) ! { + pet_store.list_pets()! +} + +fn cmd_create_pet(cmd Command) ! { + pet_store.create_pet()! +} + +fn cmd_get_pet(cmd Command) ! { + pet_store.get_pet()! +} + +fn cmd_delete_pet(cmd Command) ! { + pet_store.delete_pet()! +} + +fn cmd_list_orders(cmd Command) ! { + pet_store.list_orders()! +} + +fn cmd_get_order(cmd Command) ! { + pet_store.get_order()! +} + +fn cmd_delete_order(cmd Command) ! { + pet_store.delete_order()! +} + +fn cmd_create_user(cmd Command) ! { + pet_store.create_user()! +} diff --git a/crystallib/hero/baobab/generator/testdata/pet_store_actor/methods.v b/crystallib/hero/baobab/generator/testdata/pet_store_actor/methods.v new file mode 100644 index 000000000..897935129 --- /dev/null +++ b/crystallib/hero/baobab/generator/testdata/pet_store_actor/methods.v @@ -0,0 +1,25 @@ +module pet_store_actor + +// Method for list_pets +fn (mut actor PetStoreActor) list_pets(limit int) ! {} + +// Method for create_pet +fn (mut actor PetStoreActor) create_pet() ! {} + +// Method for get_pet +fn (mut actor PetStoreActor) get_pet(petId int) ! {} + +// Method for delete_pet +fn (mut actor PetStoreActor) delete_pet(petId int) ! {} + +// Method for list_orders +fn (mut actor PetStoreActor) list_orders() ! {} + +// Method for get_order +fn (mut actor PetStoreActor) get_order(orderId int) ! {} + +// Method for delete_order +fn (mut actor PetStoreActor) delete_order(orderId int) ! {} + +// Method for create_user +fn (mut actor PetStoreActor) create_user() ! {} diff --git a/crystallib/baobab/generator/write_object_methods.v b/crystallib/hero/baobab/generator/write_object_methods.v similarity index 97% rename from crystallib/baobab/generator/write_object_methods.v rename to crystallib/hero/baobab/generator/write_object_methods.v index cf03edc6f..1990c7c94 100644 --- a/crystallib/baobab/generator/write_object_methods.v +++ b/crystallib/hero/baobab/generator/write_object_methods.v @@ -1,6 +1,7 @@ module generator -import freeflowuniverse.crystallib.core.codemodel { CodeFile, CodeItem, Function, Import, Param, Result, Struct, StructField, Type } +import freeflowuniverse.crystallib.hero.baobab.specification {BaseObject} +import freeflowuniverse.crystallib.core.codemodel { VFile, CodeItem, Function, Import, Param, Result, Struct, StructField, Type } import freeflowuniverse.crystallib.core.codeparser import freeflowuniverse.crystallib.core.texttools import os @@ -12,7 +13,7 @@ const id_param = Param{ } } -pub fn generate_object_code(actor Struct, object BaseObject) CodeFile { +pub fn generate_object_code(actor Struct, object BaseObject) VFile { obj_name := texttools.name_fix_pascal_to_snake(object.structure.name) object_type := object.structure.name diff --git a/crystallib/baobab/generator/write_object_methods_test.v b/crystallib/hero/baobab/generator/write_object_methods_test.v similarity index 100% rename from crystallib/baobab/generator/write_object_methods_test.v rename to crystallib/hero/baobab/generator/write_object_methods_test.v diff --git a/crystallib/baobab/generator/write_object_tests.v b/crystallib/hero/baobab/generator/write_object_tests.v similarity index 96% rename from crystallib/baobab/generator/write_object_tests.v rename to crystallib/hero/baobab/generator/write_object_tests.v index 67dde8274..91483b76b 100644 --- a/crystallib/baobab/generator/write_object_tests.v +++ b/crystallib/hero/baobab/generator/write_object_tests.v @@ -1,13 +1,14 @@ module generator -import freeflowuniverse.crystallib.core.codemodel { CodeFile, CustomCode, Function, Import, Struct } +import freeflowuniverse.crystallib.core.codemodel { VFile, CustomCode, Function, Import, Struct } import freeflowuniverse.crystallib.core.codeparser +import freeflowuniverse.crystallib.hero.baobab.specification {BaseObject} import rand import freeflowuniverse.crystallib.core.texttools import os // generate_object_methods generates CRUD actor methods for a provided structure -pub fn generate_object_test_code(actor Struct, object BaseObject) !CodeFile { +pub fn generate_object_test_code(actor Struct, object BaseObject) !VFile { consts := CustomCode{"const db_dir = '\${os.home_dir()}/hero/db' const actor_name = '${actor.name}_test_actor'"} @@ -28,7 +29,7 @@ pub fn generate_object_test_code(actor Struct, object BaseObject) !CodeFile { object_type := object.structure.name // TODO: support modules outside of crystal - mut file := CodeFile{ + mut file := VFile{ name: '${object_name}_test' mod: texttools.name_fix(actor_name) imports: [ diff --git a/crystallib/baobab/osis/README.md b/crystallib/hero/baobab/osis/README.md similarity index 100% rename from crystallib/baobab/osis/README.md rename to crystallib/hero/baobab/osis/README.md diff --git a/crystallib/baobab/osis/factory.v b/crystallib/hero/baobab/osis/factory.v similarity index 100% rename from crystallib/baobab/osis/factory.v rename to crystallib/hero/baobab/osis/factory.v diff --git a/crystallib/baobab/osis/indexer.v b/crystallib/hero/baobab/osis/indexer.v similarity index 100% rename from crystallib/baobab/osis/indexer.v rename to crystallib/hero/baobab/osis/indexer.v diff --git a/crystallib/baobab/osis/indexer_generic.v b/crystallib/hero/baobab/osis/indexer_generic.v similarity index 100% rename from crystallib/baobab/osis/indexer_generic.v rename to crystallib/hero/baobab/osis/indexer_generic.v diff --git a/crystallib/baobab/osis/indexer_generic_test.v b/crystallib/hero/baobab/osis/indexer_generic_test.v similarity index 100% rename from crystallib/baobab/osis/indexer_generic_test.v rename to crystallib/hero/baobab/osis/indexer_generic_test.v diff --git a/crystallib/baobab/osis/indexer_identifier.v b/crystallib/hero/baobab/osis/indexer_identifier.v similarity index 100% rename from crystallib/baobab/osis/indexer_identifier.v rename to crystallib/hero/baobab/osis/indexer_identifier.v diff --git a/crystallib/baobab/osis/indexer_test.v b/crystallib/hero/baobab/osis/indexer_test.v similarity index 100% rename from crystallib/baobab/osis/indexer_test.v rename to crystallib/hero/baobab/osis/indexer_test.v diff --git a/crystallib/baobab/osis/model.v b/crystallib/hero/baobab/osis/model.v similarity index 100% rename from crystallib/baobab/osis/model.v rename to crystallib/hero/baobab/osis/model.v diff --git a/crystallib/baobab/osis/osis.v b/crystallib/hero/baobab/osis/osis.v similarity index 100% rename from crystallib/baobab/osis/osis.v rename to crystallib/hero/baobab/osis/osis.v diff --git a/crystallib/baobab/osis/root_object.v b/crystallib/hero/baobab/osis/root_object.v similarity index 100% rename from crystallib/baobab/osis/root_object.v rename to crystallib/hero/baobab/osis/root_object.v diff --git a/crystallib/baobab/osis/storer.v b/crystallib/hero/baobab/osis/storer.v similarity index 100% rename from crystallib/baobab/osis/storer.v rename to crystallib/hero/baobab/osis/storer.v diff --git a/crystallib/baobab/osis/storer_generic.v b/crystallib/hero/baobab/osis/storer_generic.v similarity index 100% rename from crystallib/baobab/osis/storer_generic.v rename to crystallib/hero/baobab/osis/storer_generic.v diff --git a/crystallib/hero/baobab/specification/README.md b/crystallib/hero/baobab/specification/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/crystallib/hero/baobab/specification/from_openapi.v b/crystallib/hero/baobab/specification/from_openapi.v new file mode 100644 index 000000000..217214a7e --- /dev/null +++ b/crystallib/hero/baobab/specification/from_openapi.v @@ -0,0 +1,115 @@ +module specification + +import freeflowuniverse.crystallib.core.codemodel { Struct, Function } +import freeflowuniverse.crystallib.data.jsonschema { Schema, SchemaRef } +import freeflowuniverse.crystallib.web.openapi { Operation, Parameter, OpenAPI, Components, Info, PathItem, ServerSpec } +import freeflowuniverse.crystallib.rpc.openrpc { ContentDescriptor, Error } + +// Helper function: Convert OpenAPI parameter to ContentDescriptor +fn openapi_param_to_content_descriptor(param Parameter) ContentDescriptor { + return ContentDescriptor{ + name: param.name, + summary: param.description, + description: param.description, + required: param.required, + schema: param.schema + } +} + +// Helper function: Convert OpenAPI operation to ActorMethod +fn openapi_operation_to_actor_method(op Operation, method_name string, path string) ActorMethod { + mut parameters := []ContentDescriptor{} + for param in op.parameters { + parameters << openapi_param_to_content_descriptor(param) + } + + mut result := ContentDescriptor{ + name: "result", + description: "The response of the operation.", + required: true, + schema: op.responses['200'].content['application/json'].schema + } + + mut errors := []Error{} + for status, response in op.responses { + if status.int() >= 400 { + error_schema := if response.content.len > 0 { + response.content.values()[0].schema + } else {Schema{}} + errors << Error{ + code: status.int(), + message: response.description, + data: error_schema, // Extend if error schema is defined + } + } + } + + return ActorMethod{ + name: method_name, + description: op.description, + summary: op.summary, + parameters: parameters, + result: result, + errors: errors, + } +} + +// Helper function: Convert OpenAPI schema to Struct +fn openapi_schema_to_struct(name string, schema SchemaRef) Struct { + // Assuming schema properties can be mapped to Struct fields + return Struct{ + name: name, + } +} + +// Converts OpenAPI to ActorSpecification +pub fn from_openapi(spec OpenAPI) !ActorSpecification { + mut methods := []ActorMethod{} + mut objects := []BaseObject{} + + // Extract methods from OpenAPI paths + for path, item in spec.paths { + if item.get.operation_id != '' { + methods << openapi_operation_to_actor_method(item.get, item.get.operation_id, path) + } + if item.post.operation_id != '' { + methods << openapi_operation_to_actor_method(item.post, item.post.operation_id, path) + } + if item.put.operation_id != '' { + methods << openapi_operation_to_actor_method(item.put, item.put.operation_id, path) + } + if item.delete.operation_id != '' { + methods << openapi_operation_to_actor_method(item.delete, item.delete.operation_id, path) + } + if item.patch.operation_id != '' { + methods << openapi_operation_to_actor_method(item.patch, item.patch.operation_id, path) + } + if item.head.operation_id != '' { + methods << openapi_operation_to_actor_method(item.head, item.head.operation_id, path) + } + if item.options.operation_id != '' { + methods << openapi_operation_to_actor_method(item.options, item.options.operation_id, path) + } + if item.trace.operation_id != '' { + methods << openapi_operation_to_actor_method(item.trace, item.trace.operation_id, path) + } + } + + // Extract objects from OpenAPI components.schemas + for name, schema in spec.components.schemas { + objects << BaseObject{ + structure: openapi_schema_to_struct(name, schema), + methods: []Function{}, // Add related methods if applicable + children: []Struct{}, // Add nested structures if defined + } + } + + return ActorSpecification{ + name: spec.info.title, + description: spec.info.description, + structure: Struct{}, // Assuming no top-level structure for this use case + interfaces: [.openapi], // Default to OpenAPI for input + methods: methods, + objects: objects, + } +} \ No newline at end of file diff --git a/crystallib/hero/baobab/specification/from_openapi_test.v b/crystallib/hero/baobab/specification/from_openapi_test.v new file mode 100644 index 000000000..58bff88d6 --- /dev/null +++ b/crystallib/hero/baobab/specification/from_openapi_test.v @@ -0,0 +1,567 @@ +module specification + +import freeflowuniverse.crystallib.core.codemodel { Struct, Function } +import freeflowuniverse.crystallib.rpc.openrpc { ContentDescriptor, Error } +import freeflowuniverse.crystallib.web.openapi { OpenAPI, Info, ServerSpec, Components, Operation, PathItem, PathRef } +import freeflowuniverse.crystallib.data.jsonschema {Schema, Reference, SchemaRef} + +const openapi_spec = openapi.OpenAPI{ + openapi: '3.0.3' + info: openapi.Info{ + title: 'Pet Store API' + description: 'A sample API for a pet store' + version: '1.0.0' + } + servers: [ + openapi.ServerSpec{ + url: 'https://api.petstore.example.com/v1' + description: 'Production server' + }, + openapi.ServerSpec{ + url: 'https://staging.petstore.example.com/v1' + description: 'Staging server' + } + ] + paths: { + '/pets': openapi.PathItem{ + get: openapi.Operation{ + summary: 'List all pets' + operation_id: 'listPets' + parameters: [ + openapi.Parameter{ + name: 'limit' + in_: 'query' + description: 'Maximum number of pets to return' + required: false + schema: Schema{ + typ: 'integer' + format: 'int32' + } + } + ] + responses: { + '200': openapi.ResponseSpec{ + description: 'A paginated list of pets' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Pets' + } + } + } + } + '400': openapi.ResponseSpec{ + description: 'Invalid request' + } + } + } + post: openapi.Operation{ + summary: 'Create a new pet' + operation_id: 'createPet' + request_body: openapi.RequestBody{ + required: true + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/NewPet' + } + } + } + } + responses: { + '201': openapi.ResponseSpec{ + description: 'Pet created' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Pet' + } + } + } + } + '400': openapi.ResponseSpec{ + description: 'Invalid input' + } + } + } + } + '/pets/{petId}': openapi.PathItem{ + get: openapi.Operation{ + summary: 'Get a pet by ID' + operation_id: 'getPet' + parameters: [ + openapi.Parameter{ + name: 'petId' + in_: 'path' + description: 'ID of the pet to retrieve' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '200': openapi.ResponseSpec{ + description: 'A pet' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Pet' + } + } + } + } + '404': openapi.ResponseSpec{ + description: 'Pet not found' + } + } + } + delete: openapi.Operation{ + summary: 'Delete a pet by ID' + operation_id: 'deletePet' + parameters: [ + openapi.Parameter{ + name: 'petId' + in_: 'path' + description: 'ID of the pet to delete' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '204': openapi.ResponseSpec{ + description: 'Pet deleted' + } + '404': openapi.ResponseSpec{ + description: 'Pet not found' + } + } + } + } + '/orders': openapi.PathItem{ + get: openapi.Operation{ + summary: 'List all orders' + operation_id: 'listOrders' + responses: { + '200': openapi.ResponseSpec{ + description: 'A list of orders' + content: { + 'application/json': openapi.MediaType{ + schema: Schema{ + typ: 'array' + items: SchemaRef(Reference{ + ref: '#/components/schemas/Order' + }) + } + } + } + } + } + } + } + '/orders/{orderId}': openapi.PathItem{ + get: openapi.Operation{ + summary: 'Get an order by ID' + operation_id: 'getOrder' + parameters: [ + openapi.Parameter{ + name: 'orderId' + in_: 'path' + description: 'ID of the order to retrieve' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '200': openapi.ResponseSpec{ + description: 'An order' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Order' + } + } + } + } + '404': openapi.ResponseSpec{ + description: 'Order not found' + } + } + } + delete: openapi.Operation{ + summary: 'Delete an order by ID' + operation_id: 'deleteOrder' + parameters: [ + openapi.Parameter{ + name: 'orderId' + in_: 'path' + description: 'ID of the order to delete' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '204': openapi.ResponseSpec{ + description: 'Order deleted' + } + '404': openapi.ResponseSpec{ + description: 'Order not found' + } + } + } + } + '/users': openapi.PathItem{ + post: openapi.Operation{ + summary: 'Create a user' + operation_id: 'createUser' + request_body: openapi.RequestBody{ + required: true + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/NewUser' + } + } + } + } + responses: { + '201': openapi.ResponseSpec{ + description: 'User created' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/User' + } + } + } + } + } + } + } + } + components: openapi.Components{ + schemas: { + 'Pet': SchemaRef(Schema{ + typ: 'object' + required: ['id', 'name'] + properties: { + 'id': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'name': SchemaRef(Schema{ + typ: 'string' + }) + 'tag': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + 'NewPet': SchemaRef(Schema{ + typ: 'object' + required: ['name'] + properties: { + 'name': SchemaRef(Schema{ + typ: 'string' + }) + 'tag': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + 'Pets': SchemaRef(Schema{ + typ: 'array' + items: SchemaRef(Reference{ + ref: '#/components/schemas/Pet' + }) + }) + 'Order': SchemaRef(Schema{ + typ: 'object' + required: ['id', 'petId', 'quantity', 'shipDate'] + properties: { + 'id': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'petId': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'quantity': SchemaRef(Schema{ + typ: 'integer' + format: 'int32' + }) + 'shipDate': SchemaRef(Schema{ + typ: 'string' + format: 'date-time' + }) + 'status': SchemaRef(Schema{ + typ: 'string' + enum_: ['placed', 'approved', 'delivered'] + }) + 'complete': SchemaRef(Schema{ + typ: 'boolean' + }) + } + }) + 'User': SchemaRef(Schema{ + typ: 'object' + required: ['id', 'username'] + properties: { + 'id': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'username': SchemaRef(Schema{ + typ: 'string' + }) + 'email': SchemaRef(Schema{ + typ: 'string' + }) + 'phone': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + 'NewUser': SchemaRef(Schema{ + typ: 'object' + required: ['username'] + properties: { + 'username': SchemaRef(Schema{ + typ: 'string' + }) + 'email': SchemaRef(Schema{ + typ: 'string' + }) + 'phone': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + } +} +} + +const actor_spec = specification.ActorSpecification{ + name: 'Pet Store API' + description: 'A sample API for a pet store' + interfaces: [.openapi] + methods: [ + specification.ActorMethod{ + name: 'listPets' + summary: 'List all pets' + parameters: [ + openrpc.ContentDescriptor{ + name: 'limit' + summary: 'Maximum number of pets to return' + description: 'Maximum number of pets to return' + schema: jsonschema.SchemaRef(jsonschema.Schema{ + typ: 'integer' + format: 'int32' + }) + } + ] + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + schema: jsonschema.SchemaRef(jsonschema.Reference{ + ref: '#/components/schemas/Pets' + }) + } + errors: [ + openrpc.Error{ + code: 400 + message: 'Invalid request' + } + ] + }, + specification.ActorMethod{ + name: 'createPet' + summary: 'Create a new pet' + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + } + errors: [ + openrpc.Error{ + code: 400 + message: 'Invalid input' + } + ] + }, + specification.ActorMethod{ + name: 'getPet' + summary: 'Get a pet by ID' + parameters: [ + openrpc.ContentDescriptor{ + name: 'petId' + summary: 'ID of the pet to retrieve' + description: 'ID of the pet to retrieve' + schema: jsonschema.SchemaRef(jsonschema.Schema{ + typ: 'integer' + format: 'int64' + }) + } + ] + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + schema: jsonschema.SchemaRef(jsonschema.Reference{ + ref: '#/components/schemas/Pet' + }) + } + errors: [ + openrpc.Error{ + code: 404 + message: 'Pet not found' + } + ] + }, + specification.ActorMethod{ + name: 'deletePet' + summary: 'Delete a pet by ID' + parameters: [ + openrpc.ContentDescriptor{ + name: 'petId' + summary: 'ID of the pet to delete' + description: 'ID of the pet to delete' + schema: jsonschema.SchemaRef(jsonschema.Schema{ + typ: 'integer' + format: 'int64' + }) + } + ] + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + } + errors: [ + openrpc.Error{ + code: 404 + message: 'Pet not found' + } + ] + }, + specification.ActorMethod{ + name: 'listOrders' + summary: 'List all orders' + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + schema: jsonschema.SchemaRef(jsonschema.Schema{ + typ: 'array' + items: jsonschema.Items(jsonschema.SchemaRef(jsonschema.Reference{ + ref: '#/components/schemas/Order' + })) + }) + } + }, + specification.ActorMethod{ + name: 'getOrder' + summary: 'Get an order by ID' + parameters: [ + openrpc.ContentDescriptor{ + name: 'orderId' + summary: 'ID of the order to retrieve' + description: 'ID of the order to retrieve' + schema: jsonschema.SchemaRef(jsonschema.Schema{ + typ: 'integer' + format: 'int64' + }) + } + ] + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + schema: jsonschema.SchemaRef(jsonschema.Reference{ + ref: '#/components/schemas/Order' + }) + } + errors: [ + openrpc.Error{ + code: 404 + message: 'Order not found' + } + ] + }, + specification.ActorMethod{ + name: 'deleteOrder' + summary: 'Delete an order by ID' + parameters: [ + openrpc.ContentDescriptor{ + name: 'orderId' + summary: 'ID of the order to delete' + description: 'ID of the order to delete' + schema: jsonschema.SchemaRef(jsonschema.Schema{ + typ: 'integer' + format: 'int64' + }) + } + ] + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + } + errors: [ + openrpc.Error{ + code: 404 + message: 'Order not found' + } + ] + }, + specification.ActorMethod{ + name: 'createUser' + summary: 'Create a user' + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + } + } + ] + objects: [ + specification.BaseObject{ + structure: codemodel.Struct{ + name: 'Pet' + } + }, + specification.BaseObject{ + structure: codemodel.Struct{ + name: 'NewPet' + } + }, + specification.BaseObject{ + structure: codemodel.Struct{ + name: 'Pets' + } + }, + specification.BaseObject{ + structure: codemodel.Struct{ + name: 'Order' + } + }, + specification.BaseObject{ + structure: codemodel.Struct{ + name: 'User' + } + }, + specification.BaseObject{ + structure: codemodel.Struct{ + name: 'NewUser' + } + } + ] +} + +pub fn test_from_openapi() ! { + assert from_openapi(openapi_spec)! == actor_spec +} \ No newline at end of file diff --git a/crystallib/hero/baobab/specification/from_openrpc.v b/crystallib/hero/baobab/specification/from_openrpc.v new file mode 100644 index 000000000..704426e6a --- /dev/null +++ b/crystallib/hero/baobab/specification/from_openrpc.v @@ -0,0 +1,92 @@ +module specification + +import freeflowuniverse.crystallib.rpc.openrpc { OpenRPC, Method, ContentDescriptor, Error } +import freeflowuniverse.crystallib.core.codemodel { Struct, Function } +import freeflowuniverse.crystallib.data.jsonschema { Schema, SchemaRef } + +// Helper function: Convert OpenRPC Method to ActorMethod +fn openrpc_method_to_actor_method(method Method) ActorMethod { + mut parameters := []ContentDescriptor{} + mut errors := []Error{} + + // Process parameters + for param in method.params { + parameters << param + } + + // Process errors + for err in method.errors { + errors << err + } + + // Process result + result := method.result or { + ContentDescriptor{ + name: "result" + description: "The default result of the method." + required: true + schema: SchemaRef{} // Fallback empty schema if not provided + } + } + + return ActorMethod{ + name: method.name + description: method.description + summary: method.summary + parameters: parameters + result: result + errors: errors + } +} + +// Helper function: Extract Structs from OpenRPC Components +fn extract_structs_from_openrpc(openrpc OpenRPC) []Struct { + mut structs := []Struct{} + + for schema_name, schema in openrpc.components.schemas { + mut fields := []Struct.Field{} + for field_name, field_schema in schema.properties { + fields << Struct.Field{ + name: field_name + typ: field_schema.to_code() or { panic(err) } + description: field_schema.description + required: field_name in schema.required + } + } + + structs << Struct{ + name: schema_name + description: schema.description + fields: fields + } + } + + return structs +} + +// Converts OpenRPC to ActorSpecification +pub fn from_openrpc(spec OpenRPC) !ActorSpecification { + mut methods := []ActorMethod{} + mut objects := []BaseObject{} + + // Process methods + for method in spec.methods { + methods << openrpc_method_to_actor_method(method) + } + + // Process objects (schemas) + structs := extract_structs_from_openrpc(spec) + for structure in structs { + objects << BaseObject{ + structure: structure + } + } + + return ActorSpecification{ + name: spec.info.title + description: spec.info.description + interfaces: [.openrpc] + methods: methods + objects: objects + } +} \ No newline at end of file diff --git a/crystallib/hero/baobab/specification/from_openrpc_test.v b/crystallib/hero/baobab/specification/from_openrpc_test.v new file mode 100644 index 000000000..58bff88d6 --- /dev/null +++ b/crystallib/hero/baobab/specification/from_openrpc_test.v @@ -0,0 +1,567 @@ +module specification + +import freeflowuniverse.crystallib.core.codemodel { Struct, Function } +import freeflowuniverse.crystallib.rpc.openrpc { ContentDescriptor, Error } +import freeflowuniverse.crystallib.web.openapi { OpenAPI, Info, ServerSpec, Components, Operation, PathItem, PathRef } +import freeflowuniverse.crystallib.data.jsonschema {Schema, Reference, SchemaRef} + +const openapi_spec = openapi.OpenAPI{ + openapi: '3.0.3' + info: openapi.Info{ + title: 'Pet Store API' + description: 'A sample API for a pet store' + version: '1.0.0' + } + servers: [ + openapi.ServerSpec{ + url: 'https://api.petstore.example.com/v1' + description: 'Production server' + }, + openapi.ServerSpec{ + url: 'https://staging.petstore.example.com/v1' + description: 'Staging server' + } + ] + paths: { + '/pets': openapi.PathItem{ + get: openapi.Operation{ + summary: 'List all pets' + operation_id: 'listPets' + parameters: [ + openapi.Parameter{ + name: 'limit' + in_: 'query' + description: 'Maximum number of pets to return' + required: false + schema: Schema{ + typ: 'integer' + format: 'int32' + } + } + ] + responses: { + '200': openapi.ResponseSpec{ + description: 'A paginated list of pets' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Pets' + } + } + } + } + '400': openapi.ResponseSpec{ + description: 'Invalid request' + } + } + } + post: openapi.Operation{ + summary: 'Create a new pet' + operation_id: 'createPet' + request_body: openapi.RequestBody{ + required: true + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/NewPet' + } + } + } + } + responses: { + '201': openapi.ResponseSpec{ + description: 'Pet created' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Pet' + } + } + } + } + '400': openapi.ResponseSpec{ + description: 'Invalid input' + } + } + } + } + '/pets/{petId}': openapi.PathItem{ + get: openapi.Operation{ + summary: 'Get a pet by ID' + operation_id: 'getPet' + parameters: [ + openapi.Parameter{ + name: 'petId' + in_: 'path' + description: 'ID of the pet to retrieve' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '200': openapi.ResponseSpec{ + description: 'A pet' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Pet' + } + } + } + } + '404': openapi.ResponseSpec{ + description: 'Pet not found' + } + } + } + delete: openapi.Operation{ + summary: 'Delete a pet by ID' + operation_id: 'deletePet' + parameters: [ + openapi.Parameter{ + name: 'petId' + in_: 'path' + description: 'ID of the pet to delete' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '204': openapi.ResponseSpec{ + description: 'Pet deleted' + } + '404': openapi.ResponseSpec{ + description: 'Pet not found' + } + } + } + } + '/orders': openapi.PathItem{ + get: openapi.Operation{ + summary: 'List all orders' + operation_id: 'listOrders' + responses: { + '200': openapi.ResponseSpec{ + description: 'A list of orders' + content: { + 'application/json': openapi.MediaType{ + schema: Schema{ + typ: 'array' + items: SchemaRef(Reference{ + ref: '#/components/schemas/Order' + }) + } + } + } + } + } + } + } + '/orders/{orderId}': openapi.PathItem{ + get: openapi.Operation{ + summary: 'Get an order by ID' + operation_id: 'getOrder' + parameters: [ + openapi.Parameter{ + name: 'orderId' + in_: 'path' + description: 'ID of the order to retrieve' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '200': openapi.ResponseSpec{ + description: 'An order' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Order' + } + } + } + } + '404': openapi.ResponseSpec{ + description: 'Order not found' + } + } + } + delete: openapi.Operation{ + summary: 'Delete an order by ID' + operation_id: 'deleteOrder' + parameters: [ + openapi.Parameter{ + name: 'orderId' + in_: 'path' + description: 'ID of the order to delete' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '204': openapi.ResponseSpec{ + description: 'Order deleted' + } + '404': openapi.ResponseSpec{ + description: 'Order not found' + } + } + } + } + '/users': openapi.PathItem{ + post: openapi.Operation{ + summary: 'Create a user' + operation_id: 'createUser' + request_body: openapi.RequestBody{ + required: true + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/NewUser' + } + } + } + } + responses: { + '201': openapi.ResponseSpec{ + description: 'User created' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/User' + } + } + } + } + } + } + } + } + components: openapi.Components{ + schemas: { + 'Pet': SchemaRef(Schema{ + typ: 'object' + required: ['id', 'name'] + properties: { + 'id': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'name': SchemaRef(Schema{ + typ: 'string' + }) + 'tag': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + 'NewPet': SchemaRef(Schema{ + typ: 'object' + required: ['name'] + properties: { + 'name': SchemaRef(Schema{ + typ: 'string' + }) + 'tag': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + 'Pets': SchemaRef(Schema{ + typ: 'array' + items: SchemaRef(Reference{ + ref: '#/components/schemas/Pet' + }) + }) + 'Order': SchemaRef(Schema{ + typ: 'object' + required: ['id', 'petId', 'quantity', 'shipDate'] + properties: { + 'id': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'petId': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'quantity': SchemaRef(Schema{ + typ: 'integer' + format: 'int32' + }) + 'shipDate': SchemaRef(Schema{ + typ: 'string' + format: 'date-time' + }) + 'status': SchemaRef(Schema{ + typ: 'string' + enum_: ['placed', 'approved', 'delivered'] + }) + 'complete': SchemaRef(Schema{ + typ: 'boolean' + }) + } + }) + 'User': SchemaRef(Schema{ + typ: 'object' + required: ['id', 'username'] + properties: { + 'id': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'username': SchemaRef(Schema{ + typ: 'string' + }) + 'email': SchemaRef(Schema{ + typ: 'string' + }) + 'phone': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + 'NewUser': SchemaRef(Schema{ + typ: 'object' + required: ['username'] + properties: { + 'username': SchemaRef(Schema{ + typ: 'string' + }) + 'email': SchemaRef(Schema{ + typ: 'string' + }) + 'phone': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + } +} +} + +const actor_spec = specification.ActorSpecification{ + name: 'Pet Store API' + description: 'A sample API for a pet store' + interfaces: [.openapi] + methods: [ + specification.ActorMethod{ + name: 'listPets' + summary: 'List all pets' + parameters: [ + openrpc.ContentDescriptor{ + name: 'limit' + summary: 'Maximum number of pets to return' + description: 'Maximum number of pets to return' + schema: jsonschema.SchemaRef(jsonschema.Schema{ + typ: 'integer' + format: 'int32' + }) + } + ] + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + schema: jsonschema.SchemaRef(jsonschema.Reference{ + ref: '#/components/schemas/Pets' + }) + } + errors: [ + openrpc.Error{ + code: 400 + message: 'Invalid request' + } + ] + }, + specification.ActorMethod{ + name: 'createPet' + summary: 'Create a new pet' + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + } + errors: [ + openrpc.Error{ + code: 400 + message: 'Invalid input' + } + ] + }, + specification.ActorMethod{ + name: 'getPet' + summary: 'Get a pet by ID' + parameters: [ + openrpc.ContentDescriptor{ + name: 'petId' + summary: 'ID of the pet to retrieve' + description: 'ID of the pet to retrieve' + schema: jsonschema.SchemaRef(jsonschema.Schema{ + typ: 'integer' + format: 'int64' + }) + } + ] + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + schema: jsonschema.SchemaRef(jsonschema.Reference{ + ref: '#/components/schemas/Pet' + }) + } + errors: [ + openrpc.Error{ + code: 404 + message: 'Pet not found' + } + ] + }, + specification.ActorMethod{ + name: 'deletePet' + summary: 'Delete a pet by ID' + parameters: [ + openrpc.ContentDescriptor{ + name: 'petId' + summary: 'ID of the pet to delete' + description: 'ID of the pet to delete' + schema: jsonschema.SchemaRef(jsonschema.Schema{ + typ: 'integer' + format: 'int64' + }) + } + ] + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + } + errors: [ + openrpc.Error{ + code: 404 + message: 'Pet not found' + } + ] + }, + specification.ActorMethod{ + name: 'listOrders' + summary: 'List all orders' + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + schema: jsonschema.SchemaRef(jsonschema.Schema{ + typ: 'array' + items: jsonschema.Items(jsonschema.SchemaRef(jsonschema.Reference{ + ref: '#/components/schemas/Order' + })) + }) + } + }, + specification.ActorMethod{ + name: 'getOrder' + summary: 'Get an order by ID' + parameters: [ + openrpc.ContentDescriptor{ + name: 'orderId' + summary: 'ID of the order to retrieve' + description: 'ID of the order to retrieve' + schema: jsonschema.SchemaRef(jsonschema.Schema{ + typ: 'integer' + format: 'int64' + }) + } + ] + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + schema: jsonschema.SchemaRef(jsonschema.Reference{ + ref: '#/components/schemas/Order' + }) + } + errors: [ + openrpc.Error{ + code: 404 + message: 'Order not found' + } + ] + }, + specification.ActorMethod{ + name: 'deleteOrder' + summary: 'Delete an order by ID' + parameters: [ + openrpc.ContentDescriptor{ + name: 'orderId' + summary: 'ID of the order to delete' + description: 'ID of the order to delete' + schema: jsonschema.SchemaRef(jsonschema.Schema{ + typ: 'integer' + format: 'int64' + }) + } + ] + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + } + errors: [ + openrpc.Error{ + code: 404 + message: 'Order not found' + } + ] + }, + specification.ActorMethod{ + name: 'createUser' + summary: 'Create a user' + result: openrpc.ContentDescriptor{ + name: 'result' + description: 'The response of the operation.' + } + } + ] + objects: [ + specification.BaseObject{ + structure: codemodel.Struct{ + name: 'Pet' + } + }, + specification.BaseObject{ + structure: codemodel.Struct{ + name: 'NewPet' + } + }, + specification.BaseObject{ + structure: codemodel.Struct{ + name: 'Pets' + } + }, + specification.BaseObject{ + structure: codemodel.Struct{ + name: 'Order' + } + }, + specification.BaseObject{ + structure: codemodel.Struct{ + name: 'User' + } + }, + specification.BaseObject{ + structure: codemodel.Struct{ + name: 'NewUser' + } + } + ] +} + +pub fn test_from_openapi() ! { + assert from_openapi(openapi_spec)! == actor_spec +} \ No newline at end of file diff --git a/crystallib/hero/baobab/specification/model.v b/crystallib/hero/baobab/specification/model.v new file mode 100644 index 000000000..d5a497fd8 --- /dev/null +++ b/crystallib/hero/baobab/specification/model.v @@ -0,0 +1,39 @@ +module specification + +import freeflowuniverse.crystallib.core.codemodel { Struct, Function } +import freeflowuniverse.crystallib.data.jsonschema { Schema } +import freeflowuniverse.crystallib.rpc.openrpc {ContentDescriptor, Error} + +pub struct ActorSpecification { +pub mut: + name string @[omitempty] + description string @[omitempty] + structure Struct @[omitempty] + interfaces []ActorInterface @[omitempty] + methods []ActorMethod @[omitempty] + objects []BaseObject @[omitempty] +} + +pub enum ActorInterface { + openrpc + openapi + webui + command +} + +pub struct ActorMethod { +pub: + name string @[omitempty] + description string @[omitempty] + summary string + parameters []ContentDescriptor + result ContentDescriptor + errors []Error +} + +pub struct BaseObject { +pub: + structure Struct @[omitempty] + methods []Function @[omitempty] + children []Struct @[omitempty] +} \ No newline at end of file diff --git a/crystallib/hero/baobab/specification/openrpc.v b/crystallib/hero/baobab/specification/openrpc.v new file mode 100644 index 000000000..a26cd8197 --- /dev/null +++ b/crystallib/hero/baobab/specification/openrpc.v @@ -0,0 +1,66 @@ +module specification + +import freeflowuniverse.crystallib.rpc.openrpc {OpenRPC} +import freeflowuniverse.crystallib.web.openapi {OpenAPI} + +pub fn from_openrpc(spec openrpc.OpenRPC) !ActorSpecification { + // Extract Actor metadata from OpenRPC info + // actor_name := openrpc_doc.info.title + // actor_description := openrpc_doc.info.description + + // // Generate methods + // mut methods := []ActorMethod{} + // for method in openrpc_doc.methods { + // method_code := method.to_code()! // Using provided to_code function + // methods << ActorMethod{ + // name: method.name + // func: method_code + // } + // } + + // // Generate BaseObject structs from schemas + // mut objects := []BaseObject{} + // for key, schema_ref in openrpc_doc.components.schemas { + // struct_obj := schema_ref.to_code()! // Assuming schema_ref.to_code() converts schema to Struct + // // objects << BaseObject{ + // // structure: codemodel.Struct{ + // // name: struct_obj.name + // // } + // // } + // } + + // Build the Actor struct + return ActorSpecification{ + // name: actor_name + // description: actor_description + // methods: methods + // objects: objects + } +} + + +pub fn (s ActorSpecification) to_openrpc() OpenRPC { + return OpenRPC { + + } +} + +// pub fn (actor Actor) generate_openrpc() OpenRPC { +// mut schemas := map[string]SchemaRef{} +// for obj in actor.objects { +// schemas[obj.structure.name] = jsonschema.struct_to_schema(obj.structure) +// for child in obj.children { +// schemas[child.name] = jsonschema.struct_to_schema(child) +// } +// } +// return OpenRPC{ +// info: openrpc.Info{ +// title: actor.name.title() +// version: '1.0.0' +// } +// methods: actor.methods.map(openrpc.fn_to_method(it.func)) +// components: Components{ +// schemas: schemas +// } +// } +// } diff --git a/crystallib/hero/baobab/specification/to_openapi.v b/crystallib/hero/baobab/specification/to_openapi.v new file mode 100644 index 000000000..345cf9a45 --- /dev/null +++ b/crystallib/hero/baobab/specification/to_openapi.v @@ -0,0 +1,64 @@ +module specification + +import freeflowuniverse.crystallib.core.codemodel { Struct, Function } +import freeflowuniverse.crystallib.data.jsonschema { Schema, SchemaRef } +import freeflowuniverse.crystallib.web.openapi { Operation, Parameter, OpenAPI, Components, Info, PathItem, ServerSpec } +import freeflowuniverse.crystallib.rpc.openrpc { ContentDescriptor, Error } + +// Converts ActorSpecification to OpenAPI +pub fn (s ActorSpecification) to_openapi() OpenAPI { + mut paths := map[string]PathItem{} + + // Map ActorMethods to paths + for method in s.methods { + mut op := Operation{ + summary: method.summary, + description: method.description, + operation_id: method.name, + } + + // Convert parameters to OpenAPI format + for param in method.parameters { + op.parameters << Parameter{ + name: param.name, + in_: 'query', // Default to query parameters; adjust based on function context + description: param.description, + required: param.required, + schema: param.schema, + } + } + + // Assign operation to corresponding HTTP method + // TODO: what about other verbs + paths['/${method.name}'] = PathItem{get: op} + } + + mut schemas := map[string]SchemaRef{} + for object in s.objects { + schemas[object.structure.name] = object.to_schema() + } + + return OpenAPI{ + openapi: '3.0.0', + info: Info{ + title: s.name, + summary: s.description, + description: s.description, + version: '1.0.0', + }, + servers: [ + ServerSpec{ + url: 'http://localhost:8080', + description: 'Default server', + }, + ], + paths: paths, + components: Components{ + schemas: schemas + }, + } +} + +fn (bo BaseObject) to_schema() Schema { + return Schema{} +} \ No newline at end of file diff --git a/crystallib/hero/publishing/command.v b/crystallib/hero/publishing/command.v new file mode 100644 index 000000000..6b4d55511 --- /dev/null +++ b/crystallib/hero/publishing/command.v @@ -0,0 +1,144 @@ +module publishing + +import freeflowuniverse.crystallib.core.pathlib +import cli { Command, Flag } +import os +import freeflowuniverse.crystallib.ui.console + +// path string //if location on filessytem, if exists, this has prio on git_url +// git_url string // location of where the hero scripts are +// git_pull bool // means when getting new repo will pull even when repo is already there +// git_pullreset bool // means we will force a pull and reset old content +// coderoot string //the location of coderoot if its another one +pub fn cmd_publisher(pre_func fn(Command)!) Command { + mut cmd_publisher := Command{ + name: 'publisher' + usage: ' +## Manage your publications + +example: + +hero publisher -u https://git.ourworld.tf/ourworld_holding/info_ourworld/src/branch/develop/heroscript + +If you do -gp it will pull newest book content from git and give error if there are local changes. +If you do -gr it will pull newest book content from git and overwrite local changes (careful). + + ' + description: 'create, edit, show mdbooks' + required_args: 0 + execute: cmd_publisher_execute + pre_execute: pre_func + } + + // cmd_run_add_flags(mut cmd_publisher) + + cmd_publisher.add_flag(Flag{ + flag: .string + name: 'name' + abbrev: 'n' + description: 'name of the publication.' + }) + + cmd_publisher.add_flag(Flag{ + flag: .bool + required: false + name: 'edit' + description: 'will open vscode for collections & summary.' + }) + + cmd_publisher.add_flag(Flag{ + flag: .bool + required: false + name: 'open' + abbrev: 'o' + description: 'will open the generated book.' + }) + + mut cmd_list := Command{ + sort_flags: true + name: 'list_books' + execute: cmd_publisher_list_books + description: 'will list existing mdbooks' + pre_execute: pre_func + } + + mut cmd_open := Command{ + name: 'open' + execute: cmd_publisher_open + description: 'will open the publication with the provided name' + pre_execute: pre_func + } + + cmd_open.add_flag(Flag{ + flag: .string + name: 'name' + abbrev: 'n' + description: 'name of the publication.' + }) + + cmd_publisher.add_command(cmd_list) + cmd_publisher.add_command(cmd_open) + // cmdroot.add_command(cmd_publisher) + return cmd_publisher +} + +fn cmd_publisher_list_books(cmd Command) ! { + console.print_header('Books:') + books := publisher.list_books()! + for book in books { + console.print_stdout(book.str()) + } +} + +fn cmd_publisher_open(cmd Command) ! { + name := cmd.flags.get_string('name') or { '' } + publisher.open(name)! +} + +fn cmd_publisher_execute(cmd Command) ! { + mut name := cmd.flags.get_string('name') or { '' } + + // mut url := cmd.flags.get_string('url') or { '' } + // mut path := cmd.flags.get_string('path') or { '' } + // if path.len > 0 || url.len > 0 { + // // execute the attached playbook + // mut plbook, _ := herocmds.plbook_run(cmd)! + // play(mut plbook)! + // // get name from the book.generate action + // // if name == '' { + // // mut a := plbook.action_get(actor: 'mdbook', name: 'define')! + // // name = a.params.get('name') or { '' } + // // } + // } else { + // publisher_help(cmd) + // } + + if name == '' { + console.print_debug('did not find name of book to generate, check in heroscript or specify with --name') + publisher_help(cmd) + exit(1) + } + + edit := cmd.flags.get_bool('edit') or { false } + open := cmd.flags.get_bool('open') or { false } + if edit || open { + // mdbook.book_open(name)! + } + + if edit { + // publisher.book_edit(name)! + } +} + +// fn pre_func(cmd Command) ! { +// herocmds.plbook_run(cmd)! +// } + + +fn publisher_help(cmd Command) { + console.clear() + console.print_header('Instructions for publisher:') + console.print_lf(1) + console.print_stdout(cmd.help_message()) + console.print_lf(5) +} diff --git a/crystallib/hero/publishing/play.v b/crystallib/hero/publishing/play.v new file mode 100644 index 000000000..818165923 --- /dev/null +++ b/crystallib/hero/publishing/play.v @@ -0,0 +1,124 @@ +module publishing + +import freeflowuniverse.crystallib.web.mdbook { MDBook } +import freeflowuniverse.crystallib.data.doctree +import freeflowuniverse.crystallib.core.playbook {Action} +import freeflowuniverse.crystallib.data.paramsparser {Params} +import freeflowuniverse.crystallib.develop.gittools +import freeflowuniverse.crystallib.core.pathlib +import os + +pub fn play(mut plbook playbook.PlayBook) ! { + // first lets configure are publisher + if mut action := plbook.action_get(actor: 'publisher' name:'configure') { + play_configure(mut action)! + } + + // lets add all the collections + for mut action in plbook.find(filter: 'publisher:new_collection')! { + mut p := action.params + play_new_collection(mut p)! + action.done = true + } + + // then lets export the doctree with all its collections + publisher.export_tree()! + + // now we can start defining books + for mut action in plbook.find(filter: 'book:define')! { + mut p := action.params + play_book_define(mut p)! + action.done = true + } + + // finally lets publish defined books + for mut action in plbook.find(filter: 'book:publish')! { + p := action.params + spawn play_book_publish(p) + action.done = true + } +} + +fn play_configure(mut action Action) ! { + mut buildroot := '${os.home_dir()}/hero/var/mdbuild' + mut publishroot := '${os.home_dir()}/hero/www/info' + mut coderoot := '' + // mut install := false + mut reset := false + mut pull := false + mut p := action.params + if p.exists('buildroot') { + buildroot = p.get('buildroot')! + } + if p.exists('coderoot') { + coderoot = p.get('coderoot')! + } + if p.exists('publishroot') { + publishroot = p.get('publishroot')! + } + if p.exists('reset') { + reset = p.get_default_false('reset') + } + action.done = true +} + +fn play_new_collection(mut p Params) ! { + url := p.get_default('url', '')! + path := p.get_default('path', '')! + name := p.get_default('name', '')! + reset := p.get_default_false('reset') + pull := p.get_default_false('pull') + + mut tree := publisher.tree + tree.scan_concurrent( + path: path + git_url: url + git_reset: reset + git_pull: pull + )! + publisher.tree = tree +} + +fn play_book_define(mut params Params) ! { + summary_url:= params.get_default('summary_url', '')! + summary_path := if summary_url == '' { + params.get('summary_path') or { + return error('both summary url and summary path cannot be empty') + } + } else { + get_summary_path(summary_url)! + } + + name := params.get('name')! + publisher.new_book( + name: name + title: params.get_default('title', name)! + collections: params.get_list('collections')! + summary_path: summary_path + )! +} + +fn play_book_publish(p Params) ! { + name := p.get('name')! + params := p.decode[PublishParams]()! + production := p.get_default_false('production') + publisher.publish(name, params)! +} + +fn get_summary_path(summary_url string) !string { + mut gs := gittools.get()! + repo := gs.get_repo(url:summary_url, reset: false, pull: false)! + + // get the path corresponding to the summary_url dir/file + summary_path := repo.get_path_of_url(summary_url)! + mut summary_dir := pathlib.get_dir(path: summary_path)! + + summary_file := summary_dir.file_get_ignorecase('summary.md') or { + summary_dir = summary_dir.parent()! + summary_dir.file_get_ignorecase('summary.md') or { + return error('summary from git needs to be dir or file: ${err}') + } + } + + return summary_file.path +} \ No newline at end of file diff --git a/crystallib/hero/publishing/publisher.v b/crystallib/hero/publishing/publisher.v new file mode 100644 index 000000000..0e2b1bedc --- /dev/null +++ b/crystallib/hero/publishing/publisher.v @@ -0,0 +1,128 @@ +module publishing + +import os +import freeflowuniverse.crystallib.core.pathlib {Path} +import freeflowuniverse.crystallib.osal +import freeflowuniverse.crystallib.data.doctree { Tree } +import freeflowuniverse.crystallib.web.mdbook { MDBook } + +__global ( + publisher Publisher +) + +pub struct Publisher { +pub mut: + tree Tree + books map[string]Book + root_path string = os.join_path(os.home_dir(), 'hero/publisher') +} + +// returns the directory of a given collecation +fn (p Publisher) collection_directory(name string) ?Path { + mut cols_dir := p.collections_directory() + return cols_dir.dir_get(name) or { + return none + } +} + +pub fn (p Publisher) collections_directory() pathlib.Path { + collections_path := '${p.root_path}/collections' + return pathlib.get_dir(path: collections_path) or { + panic('this should never happen ${err}') + } +} + +pub fn (p Publisher) build_directory() pathlib.Path { + build_path := '${p.root_path}/build' + return pathlib.get_dir(path: build_path) or { + panic('this should never happen ${err}') + } +} + +pub fn (p Publisher) publish_directory() pathlib.Path { + publish_path := '${p.root_path}/publish' + return pathlib.get_dir(path: publish_path) or { + panic('this should never happen ${err}') + } +} + +@[params] +pub struct PublishParams { + production bool +} + +pub fn (p Publisher) publish(name string, params PublishParams) ! { + if name !in p.books { + return error('book ${name} doesnt exist') + } + p.books[name].publish(p.publish_directory().path, params)! +} + +pub struct Book { + name string + title string + description string + path string +} + +pub fn (book Book) publish(path string, params PublishParams) ! { + os.execute_opt(' + cd ${book.path} + mdbook build --dest-dir ${path}/${book.name}' + )! +} + +pub struct NewBook { + name string + title string + description string + summary_path string + collections []string +} + +pub fn (p Publisher) new_book(book NewBook) ! { + mut mdbooks := mdbook.get()! + mut cfg := mdbooks.config()! + cfg.path_build = p.build_directory().path + cfg.path_publish = p.publish_directory().path + + mut col_paths := []string{} + for col in book.collections { + col_dir := p.collection_directory(col) or { + return error('Collection ${col} not found in publisher tree') + } + col_paths << col_dir.path + } + + _ := mdbooks.generate( + name: book.name + title: book.title + summary_path: book.summary_path + collections: col_paths + )! + publisher.books[book.name] = Book { + name: book.name + title: book.title + description: book.description + path: '${p.build_directory().path}/${book.name}' + } +} + +pub fn (book Book) print() { + println('Book: ${book.name}\n- title: ${book.title}\n- description: ${book.description}\n- path: ${book.path}') +} + +pub fn (p Publisher) open(name string) ! { + p.publish(name)! + book := p.books[name] + cmd := 'open \'${p.publish_directory().path}/${name}/index.html\'' + osal.exec(cmd: cmd)! +} + +pub fn (p Publisher) export_tree() ! { + publisher.tree.export(destination: '${publisher.root_path}/collections')! +} + +pub fn (p Publisher) list_books() ![]Book { + return p.books.values() +} \ No newline at end of file diff --git a/crystallib/osal/utils.v b/crystallib/osal/utils.v index 1f09c0a4d..ac58cf3c7 100644 --- a/crystallib/osal/utils.v +++ b/crystallib/osal/utils.v @@ -27,7 +27,7 @@ pub fn memdb_exists(key string) bool { } // Returns a logger object and allows you to specify via environment argument OSAL_LOG_LEVEL the debug level -pub fn get_logger() log.Logger { +pub fn get_logger() log.Log { log_level := env_get_default('OSAL_LOG_LEVEL', 'info') mut logger := &log.Log{} logger.set_level(match log_level.to_lower() { @@ -37,5 +37,5 @@ pub fn get_logger() log.Logger { 'error' { .error } else { .info } }) - return logger + return *logger } diff --git a/crystallib/rpc/jsonrpc/client.v b/crystallib/rpc/jsonrpc/client.v index f92c8ff7d..db630d78b 100644 --- a/crystallib/rpc/jsonrpc/client.v +++ b/crystallib/rpc/jsonrpc/client.v @@ -31,5 +31,4 @@ pub fn (mut client IJsonRpcClient) send_json_rpc[T, D](method string, data T, ti @[params] pub struct ClientConfig { address string // address of ws server - logger &log.Logger } diff --git a/crystallib/rpc/jsonrpc/generate_client.v b/crystallib/rpc/jsonrpc/generate_client.v index a2283ba8d..bc0a9d99a 100644 --- a/crystallib/rpc/jsonrpc/generate_client.v +++ b/crystallib/rpc/jsonrpc/generate_client.v @@ -1,6 +1,6 @@ module jsonrpc -import freeflowuniverse.crystallib.core.codemodel { Attribute, CodeFile, CodeItem, Function, Module, Param, Struct, StructField, Type, parse_function } +import freeflowuniverse.crystallib.core.codemodel { Attribute, VFile, CodeItem, Function, Module, Param, Struct, StructField, Type, parse_function } import freeflowuniverse.crystallib.core.texttools pub struct GenerateClientConfig { @@ -18,12 +18,12 @@ pub fn generate_client(config GenerateClientConfig) Module { } // generate_client_factory generates a factory code file with factory functions for the client -pub fn generate_client_factory(name string) !CodeFile { +pub fn generate_client_factory(name string) !VFile { mut code := []CodeItem{} code << generate_client_struct(name) code << generate_ws_factory_code(name)! - return CodeFile{ + return VFile{ mod: name imports: [] items: code diff --git a/crystallib/rpc/jsonrpc/handler.v b/crystallib/rpc/jsonrpc/handler.v index c22e43f34..c4e7d819f 100644 --- a/crystallib/rpc/jsonrpc/handler.v +++ b/crystallib/rpc/jsonrpc/handler.v @@ -9,7 +9,6 @@ import net.websocket pub struct JsonRpcHandler { pub mut: // rpcwebsocket.RpcWsServer // server for ws communication - logger &log.Logger // map of method names to procedure handlers procedures map[string]ProcedureHandler state voidptr @@ -19,10 +18,8 @@ pub mut: // decodes payload, execute procedure function, return encoded result type ProcedureHandler = fn (payload string) !string -pub fn new_handler(logger &log.Logger) !&JsonRpcHandler { - return &JsonRpcHandler{ - logger: unsafe { logger } - } +pub fn new_handler() !&JsonRpcHandler { + return &JsonRpcHandler{} } // registers procedure handlers by method name @@ -36,7 +33,7 @@ pub fn (mut handler JsonRpcHandler) handler(client &websocket.Client, message st pub fn (mut handler JsonRpcHandler) handle(message string) !string { method := jsonrpcrequest_decode_method(message)! - handler.logger.debug('handler-> handling remote procedure call to method: ${method}') + println('handler-> handling remote procedure call to method: ${method}') procedure_func := handler.procedures[method] response := procedure_func(message) or { panic(err) } return response diff --git a/crystallib/rpc/openrpc/factory.v b/crystallib/rpc/openrpc/factory.v new file mode 100644 index 000000000..135af047b --- /dev/null +++ b/crystallib/rpc/openrpc/factory.v @@ -0,0 +1,30 @@ +module openrpc + +import json +import x.json2 { Any } +import os +import freeflowuniverse.crystallib.core.pathlib +import freeflowuniverse.crystallib.data.jsonschema { Reference, decode_schemaref } + +@[params] +pub struct Params { +pub: + path string // path to openrpc.json file + text string // content of openrpc specification text +} + +pub fn new(params Params) !OpenRPC { + if params.path == '' && params.text == '' { + return OpenRPC{} + } + + if params.text != '' && params.path != '' { + return error('Either provide path or text') + } + + text := if params.path != '' { + os.read_file(params.path)! + } else { params.text } + + return decode(text)! +} \ No newline at end of file diff --git a/crystallib/rpc/openrpc/generate.v b/crystallib/rpc/openrpc/generate.v index 78928d817..0f06f0272 100644 --- a/crystallib/rpc/openrpc/generate.v +++ b/crystallib/rpc/openrpc/generate.v @@ -1,16 +1,16 @@ module openrpc -import freeflowuniverse.crystallib.core.codemodel { CodeFile, File, Function, Struct } +import freeflowuniverse.crystallib.core.codemodel { VFile, File, Function, Struct } pub struct OpenRPCCode { pub mut: openrpc_json File - handler CodeFile - handler_test CodeFile - client CodeFile - client_test CodeFile - server CodeFile - server_test CodeFile + handler VFile + handler_test VFile + client VFile + client_test VFile + server VFile + server_test VFile } pub fn (o OpenRPC) generate_code(receiver Struct, methods_map map[string]Function, objects_map map[string]Struct) !OpenRPCCode { diff --git a/crystallib/rpc/openrpc/generate_client.v b/crystallib/rpc/openrpc/generate_client.v index e75c84f8b..3118c996c 100644 --- a/crystallib/rpc/openrpc/generate_client.v +++ b/crystallib/rpc/openrpc/generate_client.v @@ -1,12 +1,12 @@ module openrpc -import freeflowuniverse.crystallib.core.codemodel { CodeFile, CodeItem, CustomCode, Function, Struct, parse_function } +import freeflowuniverse.crystallib.core.codemodel { VFile, CodeItem, CustomCode, Function, Struct, parse_function } import freeflowuniverse.crystallib.data.jsonschema import freeflowuniverse.crystallib.rpc.jsonrpc import freeflowuniverse.crystallib.core.texttools // generate_structs geenrates struct codes for schemas defined in an openrpc document -pub fn (o OpenRPC) generate_client_file(object_map map[string]Struct) !CodeFile { +pub fn (o OpenRPC) generate_client_file(object_map map[string]Struct) !VFile { name := texttools.name_fix(o.info.title) client_struct_name := '${o.info.title}Client' client_struct := jsonrpc.generate_client_struct(client_struct_name) @@ -19,7 +19,7 @@ pub fn (o OpenRPC) generate_client_file(object_map map[string]Struct) !CodeFile codemodel.parse_import('freeflowuniverse.crystallib.rpc.rpcwebsocket'), codemodel.parse_import('log')] code << methods.map(CodeItem(it)) - mut file := CodeFile{ + mut file := VFile{ name: 'client' mod: name imports: imports @@ -63,7 +63,7 @@ pub fn (cd ContentDescriptorRef) to_result() !codemodel.Result { } // generate_structs generates struct codes for schemas defined in an openrpc document -pub fn (o OpenRPC) generate_client_test_file(methods_map map[string]Function, object_map map[string]Struct) !CodeFile { +pub fn (o OpenRPC) generate_client_test_file(methods_map map[string]Function, object_map map[string]Struct) !VFile { name := texttools.name_fix(o.info.title) // client_struct_name := '${o.info.title}Client' // client_struct := jsonrpc.generate_client_struct(client_struct_name) @@ -84,7 +84,7 @@ pub fn (o OpenRPC) generate_client_test_file(methods_map map[string]Function, ob func.body = "mut client := new_ws_client(address:'ws://127.0.0.1:\${port}')!\n${func_call}" code << func } - mut file := CodeFile{ + mut file := VFile{ name: 'client_test' mod: name imports: [ diff --git a/crystallib/rpc/openrpc/generate_handler.v b/crystallib/rpc/openrpc/generate_handler.v index 6735af5c2..4c9ccd213 100644 --- a/crystallib/rpc/openrpc/generate_handler.v +++ b/crystallib/rpc/openrpc/generate_handler.v @@ -1,11 +1,11 @@ module openrpc -import freeflowuniverse.crystallib.core.codemodel { CodeFile, CodeItem, CustomCode, Function, Result, Struct, parse_import } +import freeflowuniverse.crystallib.core.codemodel { VFile, CodeItem, CustomCode, Function, Result, Struct, parse_import } import freeflowuniverse.crystallib.rpc.jsonrpc import freeflowuniverse.crystallib.core.texttools import rand -pub fn (o OpenRPC) generate_handler_file(receiver Struct, method_map map[string]Function, object_map map[string]Struct) !CodeFile { +pub fn (o OpenRPC) generate_handler_file(receiver Struct, method_map map[string]Function, object_map map[string]Struct) !VFile { name := texttools.name_fix(o.info.title) mut code := []CodeItem{} @@ -16,7 +16,7 @@ pub fn (o OpenRPC) generate_handler_file(receiver Struct, method_map map[string] parse_import('import freeflowuniverse.crystallib.core.texttools'), ] - mut file := CodeFile{ + mut file := VFile{ name: 'handler' mod: name imports: imports @@ -32,7 +32,7 @@ pub fn (o OpenRPC) generate_handler_file(receiver Struct, method_map map[string] return file } -pub fn (o OpenRPC) generate_handler_test_file(receiver Struct, method_map map[string]Function, object_map map[string]Struct) !CodeFile { +pub fn (o OpenRPC) generate_handler_test_file(receiver Struct, method_map map[string]Function, object_map map[string]Struct) !VFile { name := texttools.name_fix(o.info.title) handler_name := texttools.name_fix_pascal_to_snake(receiver.name) @@ -82,7 +82,7 @@ pub fn (o OpenRPC) generate_handler_test_file(receiver Struct, method_map map[st imports := parse_import('freeflowuniverse.crystallib.rpc.jsonrpc {new_jsonrpcrequest, jsonrpcresponse_decode, jsonrpcerror_decode}') - mut file := CodeFile{ + mut file := VFile{ name: 'handler_test' mod: name imports: [imports] diff --git a/crystallib/rpc/openrpc/generate_server.v b/crystallib/rpc/openrpc/generate_server.v index 9bb962675..2397031cc 100644 --- a/crystallib/rpc/openrpc/generate_server.v +++ b/crystallib/rpc/openrpc/generate_server.v @@ -1,6 +1,6 @@ module openrpc -import freeflowuniverse.crystallib.core.codemodel { CodeFile, CustomCode, parse_function, parse_import } +import freeflowuniverse.crystallib.core.codemodel { VFile, CustomCode, parse_function, parse_import } import freeflowuniverse.crystallib.core.texttools // pub fn (mut handler AccountantHandler) handle_ws(client &websocket.Client, message string) string { @@ -14,7 +14,7 @@ import freeflowuniverse.crystallib.core.texttools // server.run()! // } -pub fn (o OpenRPC) generate_server_file() !CodeFile { +pub fn (o OpenRPC) generate_server_file() !VFile { name := texttools.name_fix(o.info.title) mut handle_ws_fn := parse_function('pub fn (mut handler ${name.title()}Handler) handle_ws(client &websocket.Client, message string) string ')! handle_ws_fn.body = 'return handler.handle(message) or { panic(err) }' @@ -28,7 +28,7 @@ pub fn (o OpenRPC) generate_server_file() !CodeFile { server.run()!" items := handle_ws_fn - return CodeFile{ + return VFile{ mod: name name: 'server' imports: [ @@ -40,7 +40,7 @@ pub fn (o OpenRPC) generate_server_file() !CodeFile { } } -pub fn (o OpenRPC) generate_server_test_file() !CodeFile { +pub fn (o OpenRPC) generate_server_test_file() !VFile { name := texttools.name_fix(o.info.title) // mut handle_ws_fn := parse_function('pub fn (mut handler ${name.title()}Handler) handle_ws(client &websocket.Client, message string) string ')! // handle_ws_fn.body = "return handler.handle(message) or { panic(err) }" @@ -55,7 +55,7 @@ pub fn (o OpenRPC) generate_server_test_file() !CodeFile { mut test_fn := parse_function('pub fn test_wsserver() !')! test_fn.body = 'spawn run_wsserver(port)' - return CodeFile{ + return VFile{ mod: name name: 'server_test' items: [ diff --git a/crystallib/rpc/openrpc/model.v b/crystallib/rpc/openrpc/model.v index bc71efba7..96b610c5e 100644 --- a/crystallib/rpc/openrpc/model.v +++ b/crystallib/rpc/openrpc/model.v @@ -151,9 +151,10 @@ type ErrorRef = Error | Reference // TODO: handle any type for data field // Defines an application level error. pub struct Error { +pub: code int // A Number that indicates the error type that occurred. This MUST be an integer. The error codes from and including -32768 to -32000 are reserved for pre-defined errors. These pre-defined errors SHOULD be assumed to be returned from any JSON-RPC api. message string // A String providing a short description of the error. The message SHOULD be limited to a concise single sentence. - data string // A Primitive or Structured value that contains additional information about the error. This may be omitted. The value of this member is defined by the Server (e.g. detailed error information, nested errors etc.). + data SchemaRef // A Primitive or Structured value that contains additional information about the error. This may be omitted. The value of this member is defined by the Server (e.g. detailed error information, nested errors etc.). } // TODO: enforce regex requirements diff --git a/crystallib/threefold/tfrobot/vm_deploy.v b/crystallib/threefold/tfrobot/vm_deploy.v index 3755720a1..921811018 100644 --- a/crystallib/threefold/tfrobot/vm_deploy.v +++ b/crystallib/threefold/tfrobot/vm_deploy.v @@ -57,8 +57,8 @@ pub fn (mut robot TFRobot[Config]) vm_deploy(args_ VMSpecs) !VMOutput { name: node_group nodes_count: 1 free_cpu: args.cores - free_mru: args.memory - free_ssd: size + free_mru: int(args.memory) + free_ssd: int(size) }, ] vms: [ @@ -66,8 +66,8 @@ pub fn (mut robot TFRobot[Config]) vm_deploy(args_ VMSpecs) !VMOutput { name: args.name vms_count: 1 cpu: args.cores - mem: args.memory - root_size: size + mem: int(args.memory) + root_size: int(size) node_group: node_group ssh_key: 'SSH_KEY' flist: flist @@ -93,4 +93,4 @@ pub fn (mut robot TFRobot[Config]) vm_deploy(args_ VMSpecs) !VMOutput { vm_output := vm_outputs[0] return vm_output -} \ No newline at end of file +} diff --git a/crystallib/vfs/vfsourdb_core/common.v b/crystallib/vfs/ourdb_fs/common.v similarity index 97% rename from crystallib/vfs/vfsourdb_core/common.v rename to crystallib/vfs/ourdb_fs/common.v index fb0a44cf0..86a8760cf 100644 --- a/crystallib/vfs/vfsourdb_core/common.v +++ b/crystallib/vfs/ourdb_fs/common.v @@ -1,4 +1,4 @@ -module vfsourdb_core +module ourdb_fs import time diff --git a/crystallib/vfs/vfsourdb_core/data.v b/crystallib/vfs/ourdb_fs/data.v similarity index 91% rename from crystallib/vfs/vfsourdb_core/data.v rename to crystallib/vfs/ourdb_fs/data.v index 0f559b72f..5c379af70 100644 --- a/crystallib/vfs/vfsourdb_core/data.v +++ b/crystallib/vfs/ourdb_fs/data.v @@ -1,4 +1,4 @@ -module vfsourdb_core +module ourdb_fs // DataBlock represents a block of file data diff --git a/crystallib/vfs/vfsourdb_core/directory.v b/crystallib/vfs/ourdb_fs/directory.v similarity index 99% rename from crystallib/vfs/vfsourdb_core/directory.v rename to crystallib/vfs/ourdb_fs/directory.v index aba4cc817..dc08e933e 100644 --- a/crystallib/vfs/vfsourdb_core/directory.v +++ b/crystallib/vfs/ourdb_fs/directory.v @@ -1,4 +1,4 @@ -module vfsourdb_core +module ourdb_fs import time @@ -11,7 +11,7 @@ pub mut: metadata Metadata // Metadata from models_common.v children []u32 // List of child entry IDs (instead of actual entries) parent_id u32 // ID of parent directory (0 for root) - myvfs &VFS @[skip] + myvfs &OurDBFS @[skip] } pub fn (mut self Directory) save() ! { diff --git a/crystallib/vfs/vfsourdb_core/encoder.v b/crystallib/vfs/ourdb_fs/encoder.v similarity index 99% rename from crystallib/vfs/vfsourdb_core/encoder.v rename to crystallib/vfs/ourdb_fs/encoder.v index 606543d93..5841ffa97 100644 --- a/crystallib/vfs/vfsourdb_core/encoder.v +++ b/crystallib/vfs/ourdb_fs/encoder.v @@ -1,4 +1,4 @@ -module vfsourdb_core +module ourdb_fs import freeflowuniverse.crystallib.data.encoder diff --git a/crystallib/vfs/vfsourdb_core/factory.v b/crystallib/vfs/ourdb_fs/factory.v similarity index 59% rename from crystallib/vfs/vfsourdb_core/factory.v rename to crystallib/vfs/ourdb_fs/factory.v index 9de655ae6..078aa8110 100644 --- a/crystallib/vfs/vfsourdb_core/factory.v +++ b/crystallib/vfs/ourdb_fs/factory.v @@ -1,19 +1,19 @@ -module vfsourdb_core +module ourdb_fs import os import freeflowuniverse.crystallib.data.ourdb -// Factory method for creating a new VFS instance +// Factory method for creating a new OurDBFS instance @[params] pub struct VFSParams { pub: - data_dir string // Directory to store VFS data - metadata_dir string // Directory to store VFS metadata + data_dir string // Directory to store OurDBFS data + metadata_dir string // Directory to store OurDBFS metadata } -// Factory method for creating a new VFS instance -pub fn new(params VFSParams) !&VFS { +// Factory method for creating a new OurDBFS instance +pub fn new(params VFSParams) !&OurDBFS { if !os.exists(params.data_dir) { os.mkdir(params.data_dir) or { return error('Failed to create data directory: ${err}') } } @@ -21,10 +21,10 @@ pub fn new(params VFSParams) !&VFS { os.mkdir(params.metadata_dir) or { return error('Failed to create metadata directory: ${err}') } } - mut db_meta := ourdb.new(path: '${params.metadata_dir}/vfsourdb_core.db_meta')! + mut db_meta := ourdb.new(path: '${params.metadata_dir}/ourdb_fs.db_meta')! //TODO: doesn't seem to be good names mut db_data := ourdb.new(path: '${params.data_dir}/vfs_metadata.db_meta')! - mut fs := &VFS{ + mut fs := &OurDBFS{ root_id: 1 block_size: 1024 * 4 data_dir: params.data_dir diff --git a/crystallib/vfs/vfsourdb_core/file.v b/crystallib/vfs/ourdb_fs/file.v similarity index 93% rename from crystallib/vfs/vfsourdb_core/file.v rename to crystallib/vfs/ourdb_fs/file.v index 8502a7a1e..d4a88ae83 100644 --- a/crystallib/vfs/vfsourdb_core/file.v +++ b/crystallib/vfs/ourdb_fs/file.v @@ -1,4 +1,4 @@ -module vfsourdb_core +module ourdb_fs import time @@ -8,7 +8,7 @@ pub mut: metadata Metadata // Metadata from models_common.v data string // File content stored in DB parent_id u32 // ID of parent directory - myvfs &VFS @[skip] + myvfs &OurDBFS @[skip] } pub fn (mut f File) save() ! { diff --git a/crystallib/vfs/vfsourdb_core/readme.md b/crystallib/vfs/ourdb_fs/readme.md similarity index 86% rename from crystallib/vfs/vfsourdb_core/readme.md rename to crystallib/vfs/ourdb_fs/readme.md index a2a35d17a..9b17e6819 100644 --- a/crystallib/vfs/vfsourdb_core/readme.md +++ b/crystallib/vfs/ourdb_fs/readme.md @@ -1,15 +1,16 @@ -# Virtual Filesystem (VFS) as implemented on top of ourdb +# a OurDBFS: filesystem interface on top of ourbd -A Virtual Filesystem implementation in V that provides an abstraction layer over a key-value store database. The VFS manages files and directories using unique identifiers (u32) as keys and binary data ([]u8) as values. +The OurDBFS manages files and directories using unique identifiers (u32) as keys and binary data ([]u8) as values. -This is the underlying VFS for the other vfsourdb, which is in line to interface of the vfscore ## Architecture -### Storage Backend +### Storage Backend (the ourdb) + - Uses a key-value store where keys are u32 and values are []u8 (bytes) - Stores both metadata and file data in the same database - Example usage of underlying database: + ```v import crystallib.data.ourdb @@ -27,7 +28,8 @@ db_meta.delete(1)! ### Core Components -#### 1. Common Metadata (models_common.v) +#### 1. Common Metadata (common.v) + All filesystem entries (files and directories) share common metadata: ```v pub struct Metadata { @@ -44,7 +46,7 @@ pub struct Metadata { } ``` -#### 2. Files (models_file.v) +#### 2. Files (file.v) Files are represented as: ```v pub struct File { @@ -54,7 +56,7 @@ pub struct File { } ``` -#### 3. Directories (models_directory.v) +#### 3. Directories (directory.v) Directories are represented as: ```v pub struct Directory { @@ -64,7 +66,7 @@ pub struct Directory { } ``` -#### 4. Data Storage (models_data.v) +#### 4. Data Storage (data.v) File data is stored in blocks: ```v pub struct DataBlock { @@ -121,6 +123,9 @@ pub enum FileType { ### TODO Items + +> TODO: what is implemented and what not? + 1. Directory Implementation - Implement recursive listing functionality - Proper cleanup of children during deletion diff --git a/crystallib/vfs/vfsourdb_core/symlink.v b/crystallib/vfs/ourdb_fs/symlink.v similarity index 95% rename from crystallib/vfs/vfsourdb_core/symlink.v rename to crystallib/vfs/ourdb_fs/symlink.v index 03e45d92b..a5ddde96b 100644 --- a/crystallib/vfs/vfsourdb_core/symlink.v +++ b/crystallib/vfs/ourdb_fs/symlink.v @@ -1,4 +1,4 @@ -module vfsourdb_core +module ourdb_fs import time @@ -8,7 +8,7 @@ pub mut: metadata Metadata // Metadata from models_common.v target string // Path that this symlink points to parent_id u32 // ID of parent directory - myvfs &VFS @[skip] + myvfs &OurDBFS @[skip] } pub fn (mut sl Symlink) save() ! { diff --git a/crystallib/vfs/vfsourdb_core/vfs.v b/crystallib/vfs/ourdb_fs/vfs.v similarity index 87% rename from crystallib/vfs/vfsourdb_core/vfs.v rename to crystallib/vfs/ourdb_fs/vfs.v index 1405986fa..8dd34d6a3 100644 --- a/crystallib/vfs/vfsourdb_core/vfs.v +++ b/crystallib/vfs/ourdb_fs/vfs.v @@ -1,13 +1,13 @@ -module vfsourdb_core +module ourdb_fs import freeflowuniverse.crystallib.data.ourdb -// VFS represents the virtual filesystem +// OurDBFS represents the virtual filesystem @[heap] -pub struct VFS { +pub struct OurDBFS { pub mut: root_id u32 // ID of root directory block_size u32 // Size of data blocks in bytes - data_dir string // Directory to store VFS data + data_dir string // Directory to store OurDBFS data metadata_dir string //Directory where we store the metadata db_data &ourdb.OurDB // Database instance for persistent storage db_meta &ourdb.OurDB // Database instance for metadata storage @@ -15,7 +15,7 @@ pub mut: // get_root returns the root directory -pub fn (mut fs VFS) get_root() !&Directory { +pub fn (mut fs OurDBFS) get_root() !&Directory { // Try to load root directory from DB if it exists if data := fs.db_meta.get(fs.root_id) { mut loaded_root := decode_directory(data) or { @@ -36,7 +36,7 @@ pub fn (mut fs VFS) get_root() !&Directory { } // load_entry loads an entry from the database by ID and sets up parent references -fn (mut fs VFS) load_entry(id u32) !FSEntry { +fn (mut fs OurDBFS) load_entry(id u32) !FSEntry { if data := fs.db_meta.get(id) { // First byte is version, second byte indicates the type //TODO: check we dont overflow filetype (u8 in boundaries of filetype) @@ -70,7 +70,7 @@ fn (mut fs VFS) load_entry(id u32) !FSEntry { } // save_entry saves an entry to the database -pub fn (mut fs VFS) save_entry(entry FSEntry) !u32 { +pub fn (mut fs OurDBFS) save_entry(entry FSEntry) !u32 { match entry { Directory { encoded := entry.encode() @@ -94,7 +94,7 @@ pub fn (mut fs VFS) save_entry(entry FSEntry) !u32 { } // delete_entry deletes an entry from the database -pub fn (mut fs VFS) delete_entry(id u32) ! { +pub fn (mut fs OurDBFS) delete_entry(id u32) ! { fs.db_meta.delete(id) or { return error('Failed to delete entry: ${err}') } diff --git a/crystallib/vfs/vfscore/README.md b/crystallib/vfs/vfscore/README.md index 3e14bdb55..61311d6aa 100644 --- a/crystallib/vfs/vfscore/README.md +++ b/crystallib/vfs/vfscore/README.md @@ -1,6 +1,8 @@ # Virtual File System (vfscore) Module -This module provides a pluggable virtual filesystem interface with multiple implementations: +> is the interface, should not have an implementation + +This module provides a pluggable virtual filesystem interface with one default implementation done for local. 1. Local filesystem implementation (direct passthrough to OS filesystem) 2. OurDB-based implementation (stores files and metadata in OurDB) @@ -72,9 +74,9 @@ Features: - Preserves file permissions and metadata - Efficient for local file operations -### OurDB Filesystem (vfsourdb_core) +### OurDB Filesystem (ourdb_fs) -The vfsourdb_core implementation stores files and metadata in OurDB, providing a database-backed virtual filesystem. +The ourdb_fs implementation stores files and metadata in OurDB, providing a database-backed virtual filesystem. Features: - Persistent storage in OurDB diff --git a/crystallib/vfs/vfsourdb/readme.md b/crystallib/vfs/vfsourdb/readme.md index fdbb25997..7fa2f4846 100644 --- a/crystallib/vfs/vfsourdb/readme.md +++ b/crystallib/vfs/vfsourdb/readme.md @@ -1,6 +1,6 @@ # VFS Overlay of OURDb -use the vfsourdb_core implementation underneith which speaks with the ourdb +use the ourdb_fs implementation underneith which speaks with the ourdb this is basically a filesystem interface for storing files into an ourdb. diff --git a/crystallib/vfs/vfsourdb/vfsourdb.v b/crystallib/vfs/vfsourdb/vfsourdb.v index a3c287384..1fe72e30d 100644 --- a/crystallib/vfs/vfsourdb/vfsourdb.v +++ b/crystallib/vfs/vfsourdb/vfsourdb.v @@ -1,19 +1,19 @@ module vfsourdb import freeflowuniverse.crystallib.vfs.vfscore -import freeflowuniverse.crystallib.vfs.vfsourdb_core +import freeflowuniverse.crystallib.vfs.ourdb_fs import os import time // OurDBVFS represents a VFS that uses OurDB as the underlying storage pub struct OurDBVFS { mut: - core &vfsourdb_core.VFS + core &ourdb_fs.VFS } // new creates a new OurDBVFS instance pub fn new(data_dir string, metadata_dir string) !&OurDBVFS { - mut core := vfsourdb_core.new( + mut core := ourdb_fs.new( data_dir: data_dir metadata_dir: metadata_dir )! @@ -41,7 +41,7 @@ pub fn (mut self OurDBVFS) file_create(path string) !vfscore.FSEntry { pub fn (mut self OurDBVFS) file_read(path string) ![]u8 { mut entry := self.get_entry(path)! - if mut entry is vfsourdb_core.File { + if mut entry is ourdb_fs.File { content := entry.read()! return content.bytes() } @@ -50,7 +50,7 @@ pub fn (mut self OurDBVFS) file_read(path string) ![]u8 { pub fn (mut self OurDBVFS) file_write(path string, data []u8) ! { mut entry := self.get_entry(path)! - if mut entry is vfsourdb_core.File { + if mut entry is ourdb_fs.File { entry.write(data.bytestr())! } else { return error('Not a file: ${path}') @@ -121,8 +121,8 @@ pub fn (mut self OurDBVFS) link_create(target_path string, link_path string) !vf mut parent_dir := self.get_directory(parent_path)! - mut symlink := vfsourdb_core.Symlink{ - metadata: vfsourdb_core.Metadata{ + mut symlink := ourdb_fs.Symlink{ + metadata: ourdb_fs.Metadata{ id: u32(time.now().unix()) name: link_name file_type: .symlink @@ -144,7 +144,7 @@ pub fn (mut self OurDBVFS) link_create(target_path string, link_path string) !vf pub fn (mut self OurDBVFS) link_read(path string) !string { mut entry := self.get_entry(path)! - if mut entry is vfsourdb_core.Symlink { + if mut entry is ourdb_fs.Symlink { return entry.get_target()! } return error('Not a symlink: ${path}') @@ -156,7 +156,7 @@ pub fn (mut self OurDBVFS) destroy() ! { // Helper functions -fn (mut self OurDBVFS) get_entry(path string) !vfsourdb_core.FSEntry { +fn (mut self OurDBVFS) get_entry(path string) !ourdb_fs.FSEntry { if path == '/' { return self.core.get_root()! } @@ -171,7 +171,7 @@ fn (mut self OurDBVFS) get_entry(path string) !vfsourdb_core.FSEntry { for child in children { if child.metadata.name == parts[i] { match child { - vfsourdb_core.Directory { + ourdb_fs.Directory { current = child found = true break @@ -195,29 +195,29 @@ fn (mut self OurDBVFS) get_entry(path string) !vfsourdb_core.FSEntry { return current } -fn (mut self OurDBVFS) get_directory(path string) !&vfsourdb_core.Directory { +fn (mut self OurDBVFS) get_directory(path string) !&ourdb_fs.Directory { mut entry := self.get_entry(path)! - if mut entry is vfsourdb_core.Directory { + if mut entry is ourdb_fs.Directory { return &entry } return error('Not a directory: ${path}') } -fn convert_to_vfscore_entry(entry vfsourdb_core.FSEntry) vfscore.FSEntry { +fn convert_to_vfscore_entry(entry ourdb_fs.FSEntry) vfscore.FSEntry { match entry { - vfsourdb_core.Directory { + ourdb_fs.Directory { return &DirectoryEntry{ metadata: convert_metadata(entry.metadata) path: entry.metadata.name } } - vfsourdb_core.File { + ourdb_fs.File { return &FileEntry{ metadata: convert_metadata(entry.metadata) path: entry.metadata.name } } - vfsourdb_core.Symlink { + ourdb_fs.Symlink { return &SymlinkEntry{ metadata: convert_metadata(entry.metadata) path: entry.metadata.name @@ -227,7 +227,7 @@ fn convert_to_vfscore_entry(entry vfsourdb_core.FSEntry) vfscore.FSEntry { } } -fn convert_metadata(meta vfsourdb_core.Metadata) vfscore.Metadata { +fn convert_metadata(meta ourdb_fs.Metadata) vfscore.Metadata { return vfscore.Metadata{ name: meta.name file_type: match meta.file_type { diff --git a/crystallib/vfs/webdav/README.md b/crystallib/vfs/webdav/README.md new file mode 100644 index 000000000..22d6f8eec --- /dev/null +++ b/crystallib/vfs/webdav/README.md @@ -0,0 +1,35 @@ +# WebDAV Server in V + +This project implements a WebDAV server, using the `vweb` framework and modules from `crystallib`. This server allows basic file operations such as reading, writing, copying, moving, and deleting files and directories, with support for authentication and request logging. + +## Features + +- **File Operations**: Supports `GET`, `PUT`, `DELETE`, `COPY`, `MOVE`, and `MKCOL` (create directory) operations on files and directories. +- **Authentication**: Basic authentication with credentials stored in memory (`username:password`). +- **Logging**: Logs incoming requests for debugging and tracking purposes. +- **WebDAV Compliance**: Implements common WebDAV HTTP methods with responses formatted as required by WebDAV clients. +- **Customizable Middleware**: Custom middleware for authentication and logging. + +## Usage + +### Routes + +| Method | Route | Description | +|-----------|---------------|---------------------------------------------------------| +| GET | `/:path...` | Retrieves a file's contents. | +| PUT | `/:path...` | Creates or updates a file. | +| DELETE | `/:path...` | Deletes a file or directory. | +| COPY | `/:path...` | Copies a file or directory to a new location. | +| MOVE | `/:path...` | Moves a file or directory to a new location. | +| MKCOL | `/:path...` | Creates a new directory. | +| OPTIONS | `/:path...` | Lists supported WebDAV methods. | +| PROPFIND | `/:path...` | Retrieves properties of a file or directory. | + +### Authentication + +The server uses basic authentication. Set the `Authorization` header to `Basic `. + +## Configuration + +- **Root Directory**: Specify the root directory for WebDAV operations by calling `new_app(root_dir: root_path)`. +- **User Credentials**: Specify the credentials for WebDAV operations by calling `new_app(username: , password: )`. diff --git a/crystallib/vfs/webdav/app.v b/crystallib/vfs/webdav/app.v index 5f42c3402..64a96416a 100644 --- a/crystallib/vfs/webdav/app.v +++ b/crystallib/vfs/webdav/app.v @@ -2,23 +2,33 @@ module webdav import vweb import freeflowuniverse.crystallib.core.pathlib +import freeflowuniverse.crystallib.ui.console @[heap] struct App { vweb.Context - user_db map[string]string + user_db map[string]string @[required] root_dir pathlib.Path @[vweb_global] pub mut: + lock_manager LockManager + server_port int middlewares map[string][]vweb.Middleware } -fn new_app(root string) !&App { - root_dir := pathlib.get_dir(path: root, create: true)! +@[params] +pub struct AppArgs { +pub mut: + server_port int = 8080 + root_dir string @[required] + user_db map[string]string @[required] +} + +pub fn new_app(args AppArgs) !&App { + root_dir := pathlib.get_dir(path: args.root_dir, create: true)! mut app := &App{ - user_db: { - 'mario': 'hashed_password' - } + user_db: args.user_db.clone() root_dir: root_dir + server_port: args.server_port } app.middlewares['/'] << logging_middleware @@ -27,6 +37,21 @@ fn new_app(root string) !&App { return app } +@[params] +pub struct RunArgs { +pub mut: + background bool +} + +pub fn (mut app App) run(args RunArgs) { + console.print_green('Running the server on port: ${app.server_port}') + if args.background { + spawn vweb.run(app, app.server_port) + } else { + vweb.run(app, app.server_port) + } +} + pub fn (mut app App) not_found() vweb.Result { app.set_status(404, 'Not Found') return app.html('

Page not found

') diff --git a/crystallib/vfs/webdav/auth.v b/crystallib/vfs/webdav/auth.v index 4f51ac5e1..ae655e263 100644 --- a/crystallib/vfs/webdav/auth.v +++ b/crystallib/vfs/webdav/auth.v @@ -29,10 +29,8 @@ fn (mut app App) auth_middleware(mut ctx vweb.Context) bool { return false } - username := split_credentials[0] hashed_pass := split_credentials[1] - - if app.user_db[username] != hashed_pass { + if app.password != hashed_pass { ctx.set_status(401, 'Unauthorized') ctx.add_header('WWW-Authenticate', 'Basic realm="WebDAV Server"') ctx.send_response_to_client('', '') diff --git a/crystallib/vfs/webdav/factory.v b/crystallib/vfs/webdav/factory.v deleted file mode 100644 index 5f0a8e1f3..000000000 --- a/crystallib/vfs/webdav/factory.v +++ /dev/null @@ -1,19 +0,0 @@ -module webdav - -import vweb - -@[params] -pub struct WebDAVParams { -pub: - path string @[required] // root directory path for WebDAV server - port int = 8080 // port to run the server on, defaults to 8080 -} - -pub fn start(params WebDAVParams) ! { - // Implementation will be added here - mut myapp := new_app( - params.path - )! - - vweb.run(myapp, 8080) -} diff --git a/crystallib/vfs/webdav/lock.v b/crystallib/vfs/webdav/lock.v new file mode 100644 index 000000000..bbe54744a --- /dev/null +++ b/crystallib/vfs/webdav/lock.v @@ -0,0 +1,88 @@ +module webdav + +import time +import rand + + +struct Lock { + resource string + owner string + token string + depth int // 0 for a single resource, 1 for recursive + timeout int // in seconds + created_at time.Time +} + +struct LockManager { +mut: + locks map[string]Lock +} + +pub fn (mut lm LockManager) lock(resource string, owner string, depth int, timeout int) !string { + if resource in lm.locks { + // Check if the lock is still valid + existing_lock := lm.locks[resource] + if time.now().unix() - existing_lock.created_at.unix() < existing_lock.timeout { + return existing_lock.token // Resource is already locked + } + // Expired lock, remove it + lm.unlock(resource) + } + + // Generate a new lock token + token := rand.uuid_v4() + lm.locks[resource] = Lock{ + resource: resource + owner: owner + token: token + depth: depth + timeout: timeout + created_at: time.now() + } + return token +} + +pub fn (mut lm LockManager) unlock(resource string) bool { + if resource in lm.locks { + lm.locks.delete(resource) + return true + } + return false +} + +pub fn (lm LockManager) is_locked(resource string) bool { + if resource in lm.locks { + lock_ := lm.locks[resource] + // Check if lock is expired + if time.now().unix() - lock_.created_at.unix() >= lock_.timeout { + return false + } + return true + } + return false +} + +pub fn (mut lm LockManager) unlock_with_token(resource string, token string) bool { + if resource in lm.locks { + lock_ := lm.locks[resource] + if lock_.token == token { + lm.locks.delete(resource) + return true + } + } + return false +} + +fn (mut lm LockManager) lock_recursive(resource string, owner string, depth int, timeout int) !string { + if depth == 0 { + return lm.lock(resource, owner, depth, timeout) + } + // Implement logic to lock child resources if depth == 1 + return "" +} + +pub fn (mut lm LockManager) cleanup_expired_locks() { + now := time.now().unix() + lm.locks + // lm.locks = lm.locks.filter(it.value.created_at.unix() + it.value.timeout > now) +} \ No newline at end of file diff --git a/crystallib/vfs/webdav/methods.v b/crystallib/vfs/webdav/methods.v index 7a87138a6..81f2c1821 100644 --- a/crystallib/vfs/webdav/methods.v +++ b/crystallib/vfs/webdav/methods.v @@ -7,6 +7,55 @@ import encoding.xml import freeflowuniverse.crystallib.ui.console import net.urllib + +@['/:path...'; LOCK] +fn (mut app App) lock_handler(path string) vweb.Result { + // Not yet working + // TODO: Test with multiple clients + resource := app.req.url + owner := app.get_header('Owner') + if owner.len == 0 { + app.set_status(400, 'Bad Request') + return app.text('Owner header is required.') + } + + depth := if app.get_header('Depth').len > 0 { app.get_header('Depth').int() } else { 0 } + timeout := if app.get_header('Timeout').len > 0 { app.get_header('Timeout').int() } else { 3600 } + + token := app.lock_manager.lock(resource, owner, depth, timeout) or { + app.set_status(423, 'Locked') + return app.text('Resource is already locked.') + } + + app.set_status(200, 'OK') + app.add_header('Lock-Token', token) + return app.text('Lock granted with token: $token') +} + + +@['/:path...'; UNLOCK] +fn (mut app App) unlock_handler(path string) vweb.Result { + // Not yet working + // TODO: Test with multiple clients + resource := app.req.url + token := app.get_header('Lock-Token') + if token.len == 0 { + console.print_stderr('Unlock failed: `Lock-Token` header required.') + app.set_status(400, 'Bad Request') + return app.text('Lock failed: `Owner` header missing.') + } + + if app.lock_manager.unlock_with_token(resource, token) { + app.set_status(204, 'No Content') + return app.text('Lock successfully released') + } + + console.print_stderr('Resource is not locked or token mismatch.') + app.set_status(409, 'Conflict') + return app.text('Resource is not locked or token mismatch') +} + + @['/:path...'; get] fn (mut app App) get_file(path string) vweb.Result { mut file_path := pathlib.get_file(path: app.root_dir.path + path) or { return app.not_found() } @@ -146,8 +195,8 @@ fn (mut app App) move(path string) vweb.Result { fn (mut app App) mkcol(path string) vweb.Result { mut p := pathlib.get(app.root_dir.path + path) if p.exists() { - app.set_status(405, 'Method Not Allowed') - return app.text('HTTP 405: Method Not Allowed on existing entry') + app.set_status(400, 'Bad Request') + return app.text('Another collection exists at ${p.path}') } p = pathlib.get_dir(path: p.path, create: true) or { @@ -164,11 +213,11 @@ fn (mut app App) mkcol(path string) vweb.Result { fn (mut app App) options(path string) vweb.Result { app.set_status(200, 'OK') app.add_header('DAV', '1,2') - app.add_header('Allow', 'OPTIONS, PROPFIND, PROPPATCH, MKCOL, GET, HEAD, POST, PUT, DELETE, COPY, MOVE') + app.add_header('Allow', 'OPTIONS, PROPFIND, MKCOL, GET, HEAD, POST, PUT, DELETE, COPY, MOVE') app.add_header('MS-Author-Via', 'DAV') app.add_header('Access-Control-Allow-Origin', '*') - app.add_header('Access-Control-Allow-Methods', 'OPTIONS, PROPFIND, PROPPATCH, MKCOL, GET, HEAD, POST, PUT, DELETE, COPY, MOVE') - app.add_header('Access-Control-Allow-Headers', 'Depth, Authorization, Content-Type, Lock-Token, If') + app.add_header('Access-Control-Allow-Methods', 'OPTIONS, PROPFIND, MKCOL, GET, HEAD, POST, PUT, DELETE, COPY, MOVE') + app.add_header('Access-Control-Allow-Headers', 'Authorization, Content-Type') return app.text('') } diff --git a/crystallib/vfs/webdav/server_test.v b/crystallib/vfs/webdav/server_test.v index 53ee1e896..086edbd15 100644 --- a/crystallib/vfs/webdav/server_test.v +++ b/crystallib/vfs/webdav/server_test.v @@ -1,23 +1,41 @@ module webdav -import vweb import net.http import freeflowuniverse.crystallib.core.pathlib import time import encoding.base64 +import rand -fn test_get() { +fn test_run() { root_dir := '/tmp/webdav' - app := new_app(root_dir)! - spawn vweb.run(app, 8080) + mut app := new_app( + root_dir: root_dir + user_db: { + 'mario': '123' + } + )! + app.run() +} +fn test_get() { + root_dir := '/tmp/webdav' + mut app := new_app( + server_port: rand.int_in_range(8000, 9000)! + root_dir: root_dir + user_db: { + 'mario': '123' + } + )! + app.run(background: true) time.sleep(1 * time.second) file_name := 'newfile.txt' mut p := pathlib.get_file(path: '${root_dir}/${file_name}', create: true)! p.write('my new file')! - mut req := http.new_request(.get, 'http://localhost:8080/${file_name}', '') - req.add_custom_header('Authorization', 'Basic ${base64.encode_str('mario:hashed_password')}')! + mut req := http.new_request(.get, 'http://localhost:${app.server_port}/${file_name}', + '') + signature := base64.encode_str('mario:123') + req.add_custom_header('Authorization', 'Basic ${signature}')! response := req.do()! assert response.body == 'my new file' @@ -25,15 +43,22 @@ fn test_get() { fn test_put() { root_dir := '/tmp/webdav' - app := new_app(root_dir)! - spawn vweb.run(app, 8080) - + mut app := new_app( + server_port: rand.int_in_range(8000, 9000)! + root_dir: root_dir + user_db: { + 'mario': '123' + } + )! + app.run(background: true) time.sleep(1 * time.second) file_name := 'newfile_put.txt' mut data := 'my new put file' - mut req := http.new_request(.put, 'http://localhost:8080/${file_name}', data) - req.add_custom_header('Authorization', 'Basic ${base64.encode_str('mario:hashed_password')}')! + mut req := http.new_request(.put, 'http://localhost:${app.server_port}/${file_name}', + data) + signature := base64.encode_str('mario:123') + req.add_custom_header('Authorization', 'Basic ${signature}')! mut response := req.do()! mut p := pathlib.get_file(path: '${root_dir}/${file_name}')! @@ -42,8 +67,8 @@ fn test_put() { assert p.read()! == data data = 'updated data' - req = http.new_request(.put, 'http://localhost:8080/${file_name}', data) - req.add_custom_header('Authorization', 'Basic ${base64.encode_str('mario:hashed_password')}')! + req = http.new_request(.put, 'http://localhost:${app.server_port}/${file_name}', data) + req.add_custom_header('Authorization', 'Basic ${signature}')! response = req.do()! p = pathlib.get_file(path: '${root_dir}/${file_name}')! @@ -54,8 +79,14 @@ fn test_put() { fn test_copy() { root_dir := '/tmp/webdav' - app := new_app(root_dir)! - spawn vweb.run(app, 8080) + mut app := new_app( + server_port: rand.int_in_range(8000, 9000)! + root_dir: root_dir + user_db: { + 'mario': '123' + } + )! + app.run(background: true) time.sleep(1 * time.second) file_name1, file_name2 := 'newfile_copy1.txt', 'newfile_copy2.txt' @@ -63,9 +94,11 @@ fn test_copy() { data := 'file copy data' p1.write(data)! - mut req := http.new_request(.copy, 'http://localhost:8080/${file_name1}', '') - req.add_custom_header('Authorization', 'Basic ${base64.encode_str('mario:hashed_password')}')! - req.add_custom_header('Destination', 'http://localhost:8080/${file_name2}')! + mut req := http.new_request(.copy, 'http://localhost:${app.server_port}/${file_name1}', + '') + signature := base64.encode_str('mario:123') + req.add_custom_header('Authorization', 'Basic ${signature}')! + req.add_custom_header('Destination', 'http://localhost:${app.server_port}/${file_name2}')! mut response := req.do()! assert p1.exists() @@ -76,8 +109,14 @@ fn test_copy() { fn test_move() { root_dir := '/tmp/webdav' - app := new_app(root_dir)! - spawn vweb.run(app, 8080) + mut app := new_app( + server_port: rand.int_in_range(8000, 9000)! + root_dir: root_dir + user_db: { + 'mario': '123' + } + )! + app.run(background: true) time.sleep(1 * time.second) file_name1, file_name2 := 'newfile_move1.txt', 'newfile_move2.txt' @@ -85,9 +124,11 @@ fn test_move() { data := 'file move data' p.write(data)! - mut req := http.new_request(.move, 'http://localhost:8080/${file_name1}', '') - req.add_custom_header('Authorization', 'Basic ${base64.encode_str('mario:hashed_password')}')! - req.add_custom_header('Destination', 'http://localhost:8080/${file_name2}')! + mut req := http.new_request(.move, 'http://localhost:${app.server_port}/${file_name1}', + '') + signature := base64.encode_str('mario:123') + req.add_custom_header('Authorization', 'Basic ${signature}')! + req.add_custom_header('Destination', 'http://localhost:${app.server_port}/${file_name2}')! mut response := req.do()! p = pathlib.get_file(path: '${root_dir}/${file_name2}')! @@ -97,15 +138,23 @@ fn test_move() { fn test_delete() { root_dir := '/tmp/webdav' - app := new_app(root_dir)! - spawn vweb.run(app, 8080) + mut app := new_app( + server_port: rand.int_in_range(8000, 9000)! + root_dir: root_dir + user_db: { + 'mario': '123' + } + )! + app.run(background: true) time.sleep(1 * time.second) file_name := 'newfile_delete.txt' mut p := pathlib.get_file(path: '${root_dir}/${file_name}', create: true)! - mut req := http.new_request(.delete, 'http://localhost:8080/${file_name}', '') - req.add_custom_header('Authorization', 'Basic ${base64.encode_str('mario:hashed_password')}')! + mut req := http.new_request(.delete, 'http://localhost:${app.server_port}/${file_name}', + '') + signature := base64.encode_str('mario:123') + req.add_custom_header('Authorization', 'Basic ${signature}')! mut response := req.do()! assert !p.exists() @@ -113,14 +162,22 @@ fn test_delete() { fn test_mkcol() { root_dir := '/tmp/webdav' - app := new_app(root_dir)! - spawn vweb.run(app, 8080) + mut app := new_app( + server_port: rand.int_in_range(8000, 9000)! + root_dir: root_dir + user_db: { + 'mario': '123' + } + )! + app.run(background: true) time.sleep(1 * time.second) dir_name := 'newdir' - mut req := http.new_request(.mkcol, 'http://localhost:8080/${dir_name}', '') - req.add_custom_header('Authorization', 'Basic ${base64.encode_str('mario:hashed_password')}')! + mut req := http.new_request(.mkcol, 'http://localhost:${app.server_port}/${dir_name}', + '') + signature := base64.encode_str('mario:123') + req.add_custom_header('Authorization', 'Basic ${signature}')! mut response := req.do()! mut p := pathlib.get_dir(path: '${root_dir}/${dir_name}')! @@ -129,8 +186,14 @@ fn test_mkcol() { fn test_propfind() { root_dir := '/tmp/webdav' - app := new_app(root_dir)! - spawn vweb.run(app, 8080) + mut app := new_app( + server_port: rand.int_in_range(8000, 9000)! + root_dir: root_dir + user_db: { + 'mario': '123' + } + )! + app.run(background: true) time.sleep(1 * time.second) dir_name := 'newdir' @@ -143,9 +206,11 @@ fn test_propfind() { mut file2_p := pathlib.get_file(path: '${p.path}/${file2}', create: true)! mut dir1_p := pathlib.get_dir(path: '${p.path}/${dir1}', create: true)! - mut req := http.new_request(.propfind, 'http://localhost:8080/${dir_name}', '') - req.add_custom_header('Authorization', 'Basic ${base64.encode_str('mario:hashed_password')}')! + mut req := http.new_request(.propfind, 'http://localhost:${app.server_port}/${dir_name}', + '') + signature := base64.encode_str('mario:123') + req.add_custom_header('Authorization', 'Basic ${signature}')! mut response := req.do()! - assert response.body == '/newdir02024-11-07T15:36:49Z2024-11-07T15:36:49Zhttpd/unix-directoryHTTP/1.1 200 OK/newdir/dir102024-11-07T15:36:49Z2024-11-07T15:36:49Zhttpd/unix-directoryHTTP/1.1 200 OK/newdir/file1.txt02024-11-07T15:36:49Z2024-11-07T15:36:49Ztext/plainHTTP/1.1 200 OK/newdir/file2.html02024-11-07T15:36:49Z2024-11-07T15:36:49Ztext/htmlHTTP/1.1 200 OK' + assert response.status_code == 207 } diff --git a/crystallib/web/mdbook/mdbook.v b/crystallib/web/mdbook/mdbook.v index 4df52bb3d..493314b58 100644 --- a/crystallib/web/mdbook/mdbook.v +++ b/crystallib/web/mdbook/mdbook.v @@ -27,7 +27,7 @@ pub mut: foldlevel int printbook bool - summary_url string // url of the summary.md file + // summary_url string // url of the summary.md file summary_path string // can also give the path to the summary file (can be the dir or the summary itself) // doctree_url string // doctree_path string @@ -35,6 +35,8 @@ pub mut: build_path string production bool collections []string + description string + export bool // whether mdbook should be built } pub fn (mut books MDBooks[Config]) generate(args_ MDBookArgs) !&MDBook { @@ -61,24 +63,6 @@ pub fn (mut books MDBooks[Config]) generate(args_ MDBookArgs) !&MDBook { mut gs := gittools.get()! - if args.summary_url.len > 0 { - repo := gs.get_repo(url:args.summary_url, reset: false, pull: false)! - - // get the path corresponding to the summary_url dir/file - summary_path := repo.get_path_of_url(args.summary_url)! - mut summary_dir := pathlib.get_dir(path: summary_path)! - - summary_file_path := summary_dir.file_get_ignorecase('summary.md') or { - summary_dir = summary_dir.parent()! - p := summary_dir.file_get_ignorecase('summary.md') or { - return error('summary from git needs to be dir or file: ${err}') - } - p - } - - args.summary_path = summary_file_path.path - } - mut src_path := pathlib.get_dir(path: '${args.build_path}/src', create: true)! _ := pathlib.get_dir(path: '${args.build_path}/.edit', create: true)! mut collection_set := map[string]bool{} @@ -86,14 +70,22 @@ pub fn (mut books MDBooks[Config]) generate(args_ MDBookArgs) !&MDBook { // link collections from col_path to src mut p := pathlib.get_dir(path: col_path)! mut entries := p.list(dirs_only: true, recursive: false)! - for mut entry in entries.paths { - if _ := collection_set[entry.name()] { - return error('collection with name ${entry.name()} already exists') - } - - collection_set[entry.name()] = true - entry.link('${src_path.path}/${entry.name()}', true)! + + if _ := collection_set[p.name()] { + return error('collection with name ${p.name()} already exists') } + p.link('${src_path.path}/${p.name()}', true)! + + // QUESTION: why was this ever implemented per entry? + // for mut entry in entries.paths { + // if _ := collection_set[entry.name()] { + // println('collection with name ${entry.name()} already exists') + // // return error('collection with name ${entry.name()} already exists') + // } + + // collection_set[entry.name()] = true + // entry.link('${src_path.path}/${entry.name()}', true)! + // } } mut book := MDBook{ @@ -113,7 +105,7 @@ pub fn (mut books MDBooks[Config]) generate(args_ MDBookArgs) !&MDBook { if os.exists('${collection_dir_path.path}/errors.md') { summary.add_error_page(collectionname, 'errors.md') } - // // now link the collection into the build dir + // now link the exported collection into the build dir collection_dirbuild_str := '${book.path_build.path}/src/${collectionname}'.replace('~', os.home_dir()) if !pathlib.path_equal(collection_dirbuild_str, collection_dir_path.path) { @@ -190,7 +182,9 @@ You can ignore these pages, they are just to get links to work. book.template_install()! - book.generate()! + if args.export { + book.generate()! + } console.print_header(' mdbook prepared: ${book.path_build.path}') @@ -300,4 +294,4 @@ fn (mut book MDBook) summary_image_set() ! { first = false } } -} +} \ No newline at end of file diff --git a/crystallib/web/mdbook/summary.v b/crystallib/web/mdbook/summary.v index e6df29278..92d5774e9 100644 --- a/crystallib/web/mdbook/summary.v +++ b/crystallib/web/mdbook/summary.v @@ -18,7 +18,7 @@ pub struct SummaryItem { pub mut: level int description string - path string + relpath string // relative path of summary item to source collection string pagename string } @@ -86,10 +86,23 @@ pub fn (mut book MDBook) summary(production bool) !Summary { continue } - file_path := '${path_collection.path}/${pagename}' + list := path_collection.list()! + file_path_ := list.paths.filter(it.name() == pagename) + if file_path_.len == 0 { + book.error( + msg: "page find error in summary: '${line}', can't find page: ${pagename} in collection: ${path_collection_str}\n${file_path_} doesnt exist" + ) + continue + } else if file_path_.len > 1 { + book.error(msg: 'duplicate page in collection: ${pagename}') + continue + } + + file_path := file_path_[0].path + if !os.exists(file_path) || !os.is_file(file_path) { book.error( - msg: "page find error in summary: '${line}', can't find page: ${pagename} in collection: ${path_collection_str}" + msg: "page find error in summary: '${line}', can't find page: ${pagename} in collection: ${path_collection_str}\n${file_path} doesnt exist" ) continue } @@ -104,9 +117,9 @@ pub fn (mut book MDBook) summary(production bool) !Summary { summary.items << SummaryItem{ level: level - path: path description: description pagename: pagename + relpath: file_path.all_after('${book.args.build_path}/src/') // relative path of page to src dir collection: collection } } @@ -176,7 +189,7 @@ pub fn (mut self Summary) str() string { for _ in 0 .. item.level { pre += ' ' } - out << '${pre}- [${item.description}](${item.collection}/${item.pagename})' + out << '${pre}- [${item.description}](${item.relpath})' } if self.addpages.len > 0 || (!self.production && self.errors.len > 0) { diff --git a/crystallib/web/openapi/decode.v b/crystallib/web/openapi/decode.v new file mode 100644 index 000000000..093c16fd0 --- /dev/null +++ b/crystallib/web/openapi/decode.v @@ -0,0 +1,208 @@ +module openapi + +import json +import x.json2 {Any} +import freeflowuniverse.crystallib.data.jsonschema + + + +pub fn json_decode(data string) !OpenAPI { + // Decode the raw JSON into a map to allow field-specific processing + raw_map := json2.raw_decode(data)!.as_map() + + // Decode the entire OpenAPI structure using standard JSON decoding + mut spec := json.decode(OpenAPI, data)! + + // Decode all schema and schemaref fields using `jsonschema.decode_schemaref` + // 1. Process components.schemas + if 'paths' in raw_map { + mut paths := raw_map['paths'].as_map() + for key, path in paths { + spec.paths[key] = json_decode_path(spec.paths[key], path.as_map())! + } + } + + if 'components' in raw_map { + components_map := raw_map['components'].as_map() + spec.components = json_decode_components(spec.components, components_map)! + } + + // Return the fully decoded OpenAPI structure + return spec +} + +pub fn json_decode_components(components_ Components, components_map map[string]Any) !Components { + mut components := components_ + + if 'schemas' in components_map { + components.schemas = jsonschema.decode_schemaref_map(components_map['schemas'].as_map())! + } + return components +} + +pub fn json_decode_path(path_ PathItem, path_map map[string]Any) !PathItem { + mut path := path_ + + for key in path_map.keys() { + match key { + 'get' { + operation_map := path_map[key].as_map() + path.get = json_decode_operation(path.get, operation_map)! + } + 'post' { + operation_map := path_map[key].as_map() + path.post = json_decode_operation(path.post, operation_map)! + } + 'put' { + operation_map := path_map[key].as_map() + path.put = json_decode_operation(path.put, operation_map)! + } + 'delete' { + operation_map := path_map[key].as_map() + path.delete = json_decode_operation(path.delete, operation_map)! + } + 'options' { + operation_map := path_map[key].as_map() + path.options = json_decode_operation(path.options, operation_map)! + } + 'head' { + operation_map := path_map[key].as_map() + path.head = json_decode_operation(path.head, operation_map)! + } + 'patch' { + operation_map := path_map[key].as_map() + path.patch = json_decode_operation(path.patch, operation_map)! + } + 'trace' { + operation_map := path_map[key].as_map() + path.trace = json_decode_operation(path.trace, operation_map)! + } + else { + continue + } + } + } + return path +} + +pub fn json_decode_operation(operation_ Operation, operation_map map[string]Any) !Operation { + mut operation := operation_ + + if 'requestBody' in operation_map { + request_body_any := operation_map['requestBody'] + request_body_map := request_body_any.as_map() + + if 'content' in request_body_map { + mut request_body := json.decode(RequestBody, request_body_any.str())! + // mut request_body := operation.request_body as RequestBody + mut content := request_body.content.clone() + content_map := request_body_map['content'].as_map() + request_body.content = json_decode_content(content, content_map)! + operation.request_body = request_body + } + } + + if 'responses' in operation_map { + responses_map := operation_map['responses'].as_map() + for key, response_any in responses_map { + response_map := response_any.as_map() + if 'content' in response_map { + mut response := operation.responses[key] + mut content := response.content.clone() + content_map := response_map['content'].as_map() + response.content = json_decode_content(content, content_map)! + operation.responses[key] = response + } + } + } + + if 'parameters' in operation_map { + parameters_arr := operation_map['parameters'].arr() + mut parameters := []Parameter{} + for i, parameter_any in parameters_arr { + parameter_map := parameter_any.as_map() + if 'schema' in parameter_map { + mut parameter := operation.parameters[i] + parameter.schema = jsonschema.decode_schemaref(parameter_map['schema'].as_map())! + parameters << parameter + } else { + parameters << operation.parameters[i] + } + } + operation.parameters = parameters + } + + return operation +} + +fn json_decode_content(content_ map[string]MediaType, content_map map[string]Any) !map[string]MediaType { + mut content := content_.clone() + for key, item in content_map { + media_type_map := item.as_map() + schema_any := media_type_map['schema'] + mut media_type := content[key] + media_type.schema = jsonschema.decode_schemaref(schema_any.as_map())! + content[key] = media_type + } + return content +} + +// pub fn json_decode(data string) !OpenAPI { +// // Decode the raw JSON into the OpenAPI structure +// mut spec := json.decode(OpenAPI, data)! + +// data_map := json2.raw_decode(data)!.as_map() + +// // Recursively process the structure to decode SchemaRef and Schema fields +// spec = decode_recursive(spec, data_map)! + +// return spec +// } + +// fn decode_recursive[T](obj T, data_map map[string]Any) !T { +// // data_map := json2.raw_decode(data)!.as_map() + +// $for field in T.fields { +// $if field.is_array { +// val := obj.$(field.name) +// field_array := data_map[field.name].arr() +// // mut data_fmt := data.replace(action_str, '') +// // data_fmt = data.replace('define.${obj_name}', 'define') +// arr := decode_array(val, field_array)! +// obj.$(field.name) = arr +// } + + +// println('field ${field.name} ${typeof(field.typ)}') +// field_map := data_map[field.name].as_map() +// // Check if the field is of type Schema or SchemaRef +// $if field.typ is SchemaRef { +// obj.$(field.name) = jsonschema.decode_schemaref(field_map)! +// } $else $if field.typ is map[string]SchemaRef { +// // Check if the field is a map with SchemaRef or Schema as values +// obj.$(field.name) = jsonschema.decode_schemaref_map(field_map)! +// } $else { +// val := obj.$(field.name) +// obj.$(field.name) = decode_recursive(val, field_map)! +// } +// } + +// return obj +// } + +// pub fn decode_array[T](_ []T, data_arr []Any) ![]T { +// mut arr := []T{} +// for data in data_arr { +// value := T{} +// $if T is $struct { +// arr << decode_recursive(value, data.as_map())! +// } $else { +// arr << value +// } +// } +// return arr +// } + +fn (o OpenAPI) json_encode() string { + return json.encode(o).replace('ref', '\$ref') +} \ No newline at end of file diff --git a/crystallib/web/openapi/decode_test.v b/crystallib/web/openapi/decode_test.v new file mode 100644 index 000000000..d2f845457 --- /dev/null +++ b/crystallib/web/openapi/decode_test.v @@ -0,0 +1,396 @@ +module openapi + +import os +import json +import freeflowuniverse.crystallib.data.jsonschema {Schema, Reference, SchemaRef} + +const spec_path = '${os.dir(@FILE)}/testdata/openapi.json' +const spec_json = os.read_file(spec_path) or {panic(err)} + +const spec = openapi.OpenAPI{ + openapi: '3.0.3' + info: openapi.Info{ + title: 'Pet Store API' + description: 'A sample API for a pet store' + version: '1.0.0' + } + servers: [ + openapi.Server{ + url: 'https://api.petstore.example.com/v1' + description: 'Production server' + }, + openapi.Server{ + url: 'https://staging.petstore.example.com/v1' + description: 'Staging server' + } + ] + paths: { + '/pets': openapi.PathItem{ + get: openapi.Operation{ + summary: 'List all pets' + operation_id: 'listPets' + parameters: [ + openapi.Parameter{ + name: 'limit' + in_: 'query' + description: 'Maximum number of pets to return' + required: false + schema: Schema{ + typ: 'integer' + format: 'int32' + } + } + ] + responses: { + '200': openapi.Response{ + description: 'A paginated list of pets' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Pets' + } + } + } + } + '400': openapi.Response{ + description: 'Invalid request' + } + } + } + post: openapi.Operation{ + summary: 'Create a new pet' + operation_id: 'createPet' + request_body: openapi.RequestBody{ + required: true + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/NewPet' + } + } + } + } + responses: { + '201': openapi.Response{ + description: 'Pet created' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Pet' + } + } + } + } + '400': openapi.Response{ + description: 'Invalid input' + } + } + } + } + '/pets/{petId}': openapi.PathItem{ + get: openapi.Operation{ + summary: 'Get a pet by ID' + operation_id: 'getPet' + parameters: [ + openapi.Parameter{ + name: 'petId' + in_: 'path' + description: 'ID of the pet to retrieve' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '200': openapi.Response{ + description: 'A pet' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Pet' + } + } + } + } + '404': openapi.Response{ + description: 'Pet not found' + } + } + } + delete: openapi.Operation{ + summary: 'Delete a pet by ID' + operation_id: 'deletePet' + parameters: [ + openapi.Parameter{ + name: 'petId' + in_: 'path' + description: 'ID of the pet to delete' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '204': openapi.Response{ + description: 'Pet deleted' + } + '404': openapi.Response{ + description: 'Pet not found' + } + } + } + } + '/orders': openapi.PathItem{ + get: openapi.Operation{ + summary: 'List all orders' + operation_id: 'listOrders' + responses: { + '200': openapi.Response{ + description: 'A list of orders' + content: { + 'application/json': openapi.MediaType{ + schema: Schema{ + typ: 'array' + items: SchemaRef(Reference{ + ref: '#/components/schemas/Order' + }) + } + } + } + } + } + } + } + '/orders/{orderId}': openapi.PathItem{ + get: openapi.Operation{ + summary: 'Get an order by ID' + operation_id: 'getOrder' + parameters: [ + openapi.Parameter{ + name: 'orderId' + in_: 'path' + description: 'ID of the order to retrieve' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '200': openapi.Response{ + description: 'An order' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Order' + } + } + } + } + '404': openapi.Response{ + description: 'Order not found' + } + } + } + delete: openapi.Operation{ + summary: 'Delete an order by ID' + operation_id: 'deleteOrder' + parameters: [ + openapi.Parameter{ + name: 'orderId' + in_: 'path' + description: 'ID of the order to delete' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '204': openapi.Response{ + description: 'Order deleted' + } + '404': openapi.Response{ + description: 'Order not found' + } + } + } + } + '/users': openapi.PathItem{ + post: openapi.Operation{ + summary: 'Create a user' + operation_id: 'createUser' + request_body: openapi.RequestBody{ + required: true + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/NewUser' + } + } + } + } + responses: { + '201': openapi.Response{ + description: 'User created' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/User' + } + } + } + } + } + } + } + } + components: openapi.Components{ + schemas: { + 'Pet': SchemaRef(Schema{ + typ: 'object' + required: ['id', 'name'] + properties: { + 'id': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'name': SchemaRef(Schema{ + typ: 'string' + }) + 'tag': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + 'NewPet': SchemaRef(Schema{ + typ: 'object' + required: ['name'] + properties: { + 'name': SchemaRef(Schema{ + typ: 'string' + }) + 'tag': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + 'Pets': SchemaRef(Schema{ + typ: 'array' + items: SchemaRef(Reference{ + ref: '#/components/schemas/Pet' + }) + }) + 'Order': SchemaRef(Schema{ + typ: 'object' + required: ['id', 'petId', 'quantity', 'shipDate'] + properties: { + 'id': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'petId': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'quantity': SchemaRef(Schema{ + typ: 'integer' + format: 'int32' + }) + 'shipDate': SchemaRef(Schema{ + typ: 'string' + format: 'date-time' + }) + 'status': SchemaRef(Schema{ + typ: 'string' + enum_: ['placed', 'approved', 'delivered'] + }) + 'complete': SchemaRef(Schema{ + typ: 'boolean' + }) + } + }) + 'User': SchemaRef(Schema{ + typ: 'object' + required: ['id', 'username'] + properties: { + 'id': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'username': SchemaRef(Schema{ + typ: 'string' + }) + 'email': SchemaRef(Schema{ + typ: 'string' + }) + 'phone': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + 'NewUser': SchemaRef(Schema{ + typ: 'object' + required: ['username'] + properties: { + 'username': SchemaRef(Schema{ + typ: 'string' + }) + 'email': SchemaRef(Schema{ + typ: 'string' + }) + 'phone': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + } +} +} + +pub fn testsuite_begin() {} + +fn test_decode() { + decoded := json_decode(spec_json)! + + assert decoded.openapi == spec.openapi + assert decoded.info == spec.info + assert decoded.servers == spec.servers + for key, path in decoded.paths { + assert path.ref == spec.paths[key].ref, 'Paths ${key} dont match.' + assert path.summary == spec.paths[key].summary, 'Paths ${key} dont match.' + assert path.description == spec.paths[key].description, 'Paths ${key} dont match.' + match_operations(path.get, spec.paths[key].get) + match_operations(path.put, spec.paths[key].put) + match_operations(path.post, spec.paths[key].post) + match_operations(path.delete, spec.paths[key].delete) + } + assert decoded.webhooks == spec.webhooks + for key, schema in decoded.components.schemas { + assert schema == spec.components.schemas[key], 'Schemas ${key} dont match.' + } + assert decoded.components == spec.components + assert decoded.security == spec.security +} + +fn match_operations(a Operation, b Operation) { + println(a.responses['200'].content['application/json'].schema) + assert a.tags == b.tags, 'Tags do not match.' + assert a.summary == b.summary, 'Summary does not match.' + assert a.description == b.description, 'Description does not match.' + assert a.external_docs == b.external_docs, 'External documentation does not match.' + assert a.operation_id == b.operation_id, 'Operation ID does not match.' + assert a.parameters == b.parameters, 'Parameters do not match.' + assert a.request_body == b.request_body, 'Request body does not match.' + assert a.responses == b.responses, 'Responses do not match.' + assert a.callbacks == b.callbacks, 'Callbacks do not match.' + assert a.deprecated == b.deprecated, 'Deprecated flag does not match.' + assert a.security == b.security, 'Security requirements do not match.' + assert a.servers == b.servers, 'Servers do not match.' +} \ No newline at end of file diff --git a/crystallib/web/openapi/gen/README.md b/crystallib/web/openapi/gen/README.md new file mode 100644 index 000000000..0fb734721 --- /dev/null +++ b/crystallib/web/openapi/gen/README.md @@ -0,0 +1,120 @@ +## OpenAPI Code Generation Module + + +### Way structure definitions are written and arranged + +Object schemas are defined in an OpenAPI Specification, which define the structure of data passed as parameters to a API Call, and data returned by the calls. + +These schemas therefore require data structures that need to be defined as V `struct`s in code. + +Object schemas defined in the components field of the OpenAPI Specification are assumed to be 'common' to API calls defined in the specification. The `struct`s representing these common object schemas are therefore defined in a `model.v` file. + +After that, the schemas defined in the path operations are generated alongside the Client API Methods they belong to. + +`openapi.json` +```json +{ + "components": { + "schemas": { + "Person": {} + } + }, + "paths": { + "/new_person": { + "post": { + "parameters": [ + { + "name": "person_args", + "schema": { + "type": "object" + } + } + ] + } + } + } +} +``` + +`model.v` +``` +struct Person{} +``` + +`methods.v` +``` +struct NewPersonArgs {} + +fn new_person(person_args NewPersonArgs) Person {} +``` + +## OpenAPI Handler Generator + +This project provides a utility to generate VLang handler functions and a main routing function based on an OpenAPI specification. It simplifies server-side integration by automatically creating boilerplate code for handling requests and mapping them to business logic. + +### Features + +- **Automatic Handler Generation:** Generates individual handlers for each operation in the OpenAPI spec. +- **Main Router Function:** Creates a centralized function to route incoming requests based on operation IDs. +- **Error Handling:** Includes basic error handling for invalid input and unrecognized operations. +- **Customizable Output:** Easily extendable to handle query parameters, headers, and more. + +### How It Works + +1. Parse the OpenAPI specification. +2. For each path and operation in the spec: + - Generate an individual handler function. + - Add a corresponding case in the main routing function. +3. Combine everything into a single V file for easy integration. + +#### Example Workflow + +1. **Input:** Provide an OpenAPI spec (e.g., `petstore.yaml`). +2. **Generated Output:** + + - **Individual Operation Handlers:** + ```v + fn (mut actor Actor) handle_listPets(data string) !string { + println('Handling listPets with data: $data') + params := json.decode(ListPetParams, data) or { return error("Invalid input data: $err") } + result := actor.data_store.list_pets(params) + return json.encode(result) + } + ``` + + - **Main Routing Function:** + ```v + pub fn (mut h OpenAPIHandler) handle(req Request) !Response { + match req.operation.operation_id { + "listPets" { + println("Handling listPets for GET /pets") + response := h.actor.handle_listPets(req.body) or { + return Response{ status: http.Status.internal_server_error, body: "Internal server error: $err" } + } + return Response{ status: http.Status.ok, body: response } + } + else { + return error("Unknown operation: ${req.operation.operation_id}") + } + } + } + ``` + +### Usage + +#### Input Requirements + +- **OpenAPI Specification**: A valid OpenAPI 3.0 specification, either in JSON or YAML format. +- **V Structs**: Ensure that parameter and schema structs (e.g., `ListPetParams`, `NewPet`) are defined in your project. + +#### Example Code to Generate Handlers + +```v +import your_openapi_parser_module + +fn main() { + spec := your_openapi_parser_module.parse('path/to/openapi.yaml')! + generated_code := openapi_to_handler_file(spec) + os.write_file('generated_handlers.v', generated_code) or { panic(err) } + println('Handlers successfully generated!') +} \ No newline at end of file diff --git a/crystallib/core/openapi/gen/client.v b/crystallib/web/openapi/gen/client.v similarity index 100% rename from crystallib/core/openapi/gen/client.v rename to crystallib/web/openapi/gen/client.v diff --git a/crystallib/core/openapi/gen/factory.v b/crystallib/web/openapi/gen/factory.v similarity index 100% rename from crystallib/core/openapi/gen/factory.v rename to crystallib/web/openapi/gen/factory.v diff --git a/crystallib/core/openapi/gen/factory_test.v b/crystallib/web/openapi/gen/factory_test.v similarity index 100% rename from crystallib/core/openapi/gen/factory_test.v rename to crystallib/web/openapi/gen/factory_test.v diff --git a/crystallib/core/openapi/gen/generator.v b/crystallib/web/openapi/gen/generator.v similarity index 93% rename from crystallib/core/openapi/gen/generator.v rename to crystallib/web/openapi/gen/generator.v index 259c5579e..eb1af785c 100644 --- a/crystallib/core/openapi/gen/generator.v +++ b/crystallib/web/openapi/gen/generator.v @@ -1,7 +1,7 @@ module gen import net.http -import freeflowuniverse.crystallib.core.codemodel { CodeFile, CodeItem, Struct, Type } +import freeflowuniverse.crystallib.core.codemodel { VFile, CodeItem, Struct, Type } import freeflowuniverse.crystallib.core.texttools import freeflowuniverse.crystallib.ui.console @@ -13,8 +13,8 @@ pub mut: generated_methods []string } -fn (mut gen ClientGenerator) generate_client() CodeFile { - return CodeFile{ +fn (mut gen ClientGenerator) generate_client() VFile { + return VFile{ name: 'client' mod: '${gen.api_name}_client' imports: [] @@ -75,7 +75,7 @@ fn generate_client_config() Struct { } } -fn (mut gen ClientGenerator) generate_factory() CodeFile { +fn (mut gen ClientGenerator) generate_factory() VFile { client_name := texttools.name_fix(gen.api_name) client_struct := gen.generate_client_struct() config_struct := generate_client_config() @@ -83,7 +83,7 @@ fn (mut gen ClientGenerator) generate_factory() CodeFile { heroplay_function := gen.heroplay_function(client_struct) config_interactive_function := gen.config_interactive_function(client_struct) - return CodeFile{ + return VFile{ name: 'factory' mod: '${gen.api_name}_client' imports: [] @@ -205,15 +205,15 @@ fn (mut gen ClientGenerator) config_interactive_function(client Struct) codemode } } -fn (mut gen ClientGenerator) generate_model(structs []Struct) !CodeFile { - return CodeFile{ +fn (mut gen ClientGenerator) generate_model(structs []Struct) !VFile { + return VFile{ name: 'model' mod: '${gen.api_name}_client' items: structs.map(CodeItem(it)) } } -fn (mut gen ClientGenerator) generate_methods(paths []Path) !CodeFile { +fn (mut gen ClientGenerator) generate_methods(paths []Path) !VFile { mut code := []CodeItem{} for path in paths { for operation in path.operations { @@ -223,7 +223,7 @@ fn (mut gen ClientGenerator) generate_methods(paths []Path) !CodeFile { code << gen.generate_client_method()! } } - return CodeFile{ + return VFile{ name: 'methods' items: code } diff --git a/crystallib/core/openapi/gen/generator_test.v b/crystallib/web/openapi/gen/generator_test.v similarity index 97% rename from crystallib/core/openapi/gen/generator_test.v rename to crystallib/web/openapi/gen/generator_test.v index 2e3964bc6..822325644 100644 --- a/crystallib/core/openapi/gen/generator_test.v +++ b/crystallib/web/openapi/gen/generator_test.v @@ -45,14 +45,14 @@ fn test_generate_model() { assert (model_file.items[0] as Struct).name == 'SomeModel' } -// fn (mut gen ClientGenerator) generate_model(structs []Struct) !CodeFile { -// return CodeFile{ +// fn (mut gen ClientGenerator) generate_model(structs []Struct) !VFile { +// return VFile{ // name: 'model' // items: structs.map(CodeItem(it)) // } // } -// fn (mut gen ClientGenerator) generate_methods(paths []Path) !CodeFile { +// fn (mut gen ClientGenerator) generate_methods(paths []Path) !VFile { // mut code := []CodeItem{} // for path in paths { // for operation in path.operations { @@ -62,7 +62,7 @@ fn test_generate_model() { // code << gen.generate_client_method()! // } // } -// return CodeFile{ +// return VFile{ // name: 'methods' // items: code // } diff --git a/crystallib/web/openapi/gen/handler.v b/crystallib/web/openapi/gen/handler.v new file mode 100644 index 000000000..f620072f3 --- /dev/null +++ b/crystallib/web/openapi/gen/handler.v @@ -0,0 +1,75 @@ +module generation + +import freeflowuniverse.crystallib.web.openapi {OpenAPI} + +pub fn openapi_to_handler_file(spec OpenAPI) string { + mut operation_handlers := []string{} + mut routes := []string{} + + // Iterate over OpenAPI paths and operations + for path, path_item in spec.paths { + for method_name, operation in path_item.methods { + if operation is openapi.Operation { + operation_id := operation.operation_id + params := operation.parameters.map(it.name).join(', ') + + // Generate individual handler + handler := generate_individual_handler(method_name, operation_id, params) + operation_handlers << handler + + // Generate route case + route := generate_route_case(method_name, path, operation_id) + routes << route + } + } + } + + // Combine the generated handlers and main router into a single file + return [ + '// AUTO-GENERATED FILE - DO NOT EDIT MANUALLY', + '', + 'pub struct OpenAPIHandler {', + ' mut:', + ' actor Actor', + '}', + '', + operation_handlers.join('\n\n'), + '', + 'pub fn (mut h OpenAPIHandler) handle(req Request) !Response {', + ' match req.operation.operation_id {', + routes.join('\n'), + ' else {', + ' return error("Unknown operation: ${req.operation.operation_id}")', + ' }', + ' }', + '}', + ].join('\n') +} + +// Helper function to generate individual handlers +fn generate_individual_handler(method string, operation_id string, params string) string { + mut handler := '// Handler for $operation_id\n' + handler += "fn (mut actor Actor) handle_$operation_id(data string) !string {\n" + handler += " println('Handling $operation_id with data: \$data')\n" + if params.len > 0 { + handler += ' params := json.decode($params, data) or { return error("Invalid input data: \$err") }\n' + handler += ' result := actor.data_store.$operation_id(params)\n' + } else { + handler += ' result := actor.data_store.$operation_id()\n' + } + handler += ' return json.encode(result)\n' + handler += '}' + return handler +} + +// Helper function to generate a case block for the main router +fn generate_route_case(method string, path string, operation_id string) string { + mut case_block := ' "${operation_id}" {' + case_block += '\n println("Handling $operation_id for ${method}")' + case_block += '\n response := h.actor.handle_$operation_id(req.body) or {' + case_block += '\n return Response{ status: http.Status.internal_server_error, body: "Internal server error: $err" }' + case_block += '\n }' + case_block += '\n return Response{ status: http.Status.ok, body: response }' + case_block += '\n }' + return case_block +} \ No newline at end of file diff --git a/crystallib/core/openapi/gen/templates/factory.v_ b/crystallib/web/openapi/gen/templates/factory.v_ similarity index 100% rename from crystallib/core/openapi/gen/templates/factory.v_ rename to crystallib/web/openapi/gen/templates/factory.v_ diff --git a/crystallib/core/openapi/generator.v b/crystallib/web/openapi/generator.v similarity index 100% rename from crystallib/core/openapi/generator.v rename to crystallib/web/openapi/generator.v diff --git a/crystallib/web/openapi/handler.v b/crystallib/web/openapi/handler.v new file mode 100644 index 000000000..85cc2a521 --- /dev/null +++ b/crystallib/web/openapi/handler.v @@ -0,0 +1,53 @@ +module openapi + +import net.http {CommonHeader} +import x.json2 {Any} + +pub struct Request { +pub: + path string // The requested path + method string // HTTP method (e.g., GET, POST) + key string + body string // Request body + operation Operation + arguments map[string]Any + parameters map[string]string + header http.Header @[omitempty; str: skip; json: '-']// Request headers +} + +pub struct Response { +pub mut: + status http.Status // HTTP status + body string // Response body + header http.Header @[omitempty; str: skip; json:'-']// Response headers +} + +pub interface IHandler { +mut: + handle(Request) !Response +} + +pub struct Handler { +pub: + routes map[string]fn (Request) !Response // Map of route handlers +} + +// Handle a request and return a response +pub fn (handler Handler) handle(request Request) !Response { + // Match the route based on the request path + if route_handler := handler.routes[request.path] { + // Call the corresponding route handler + return route_handler(request) + } + + // Return 404 if no route matches + return Response{ + status: .not_found + body: 'Not Found' + header: http.new_header( + key: CommonHeader.content_type, + value: 'text/plain' + ) + } +} + diff --git a/crystallib/web/openapi/model.v b/crystallib/web/openapi/model.v new file mode 100644 index 000000000..6e08b7457 --- /dev/null +++ b/crystallib/web/openapi/model.v @@ -0,0 +1,257 @@ +module openapi + +import json +import x.json2 { Any } +import freeflowuniverse.crystallib.data.jsonschema {Schema, Reference, SchemaRef} + +// todo: report bug: when comps is optional, doesnt work +pub struct OpenAPI { +pub mut: + openapi string @[required] // This string MUST be the version number of the OpenAPI Specification that the OpenAPI document uses. The openapi field SHOULD be used by tooling to interpret the OpenAPI document. This is not related to the API info.version string. + info Info @[required] // Provides metadata about the API. The metadata MAY be used by tooling as required. + json_schema_dialect string // The default value for the $schema keyword within Schema Objects contained within this OAS document. This MUST be in the form of a URI. + servers []ServerSpec // An array of ServerSpec Objects, which provide connectivity information to a target server. If the servers property is not provided, or is an empty array, the default value would be a ServerSpec Object with a url value of /. + paths map[string]PathItem // The available paths and operations for the API. + webhooks map[string]PathRef // The incoming webhooks that MAY be received as part of this API and that the API consumer MAY choose to implement. Closely related to the callbacks feature, this section describes requests initiated other than by an API call, for example by an out of band registration. The key name is a unique string to refer to each webhook, while the (optionally referenced) Path Item Object describes a request that may be initiated by the API provider and the expected responses. An example is available. + components Components // An element to hold various schemas for the document. + security []SecurityRequirement // A declaration of which security mechanisms can be used across the API. The list of values includes alternative security requirement objects that can be used. Only one of the security requirement objects need to be satisfied to authorize a request. Individual operations can override this definition. To make security optional, an empty security requirement ({}) can be included in the array. + tags []Tag // A list of tags used by the document with additional metadata. The order of the tags can be used to reflect on their order by the parsing tools. Not all tags that are used by the Operation Object must be declared. The tags that are not declared MAY be organized randomly or based on the tools’ logic. Each tag name in the list MUST be unique. + external_docs ExternalDocumentation // Additional external documentation. +} + +pub fn (spec OpenAPI) plain() string { + return '${spec}'.split('\n').filter(!it.contains('Option(none)')).join('\n') +} + +// ``` +// { +// "title": "Sample Pet Store App", +// "summary": "A pet store manager.", +// "description": "This is a sample server for a pet store.", +// "termsOfService": "https://example.com/terms/", +// "contact": { +// "name": "API Support", +// "url": "https://www.example.com/support", +// "email": "support@example.com" +// }, +// "license": { +// "name": "Apache 2.0", +// "url": "https://www.apache.org/licenses/LICENSE-2.0.html" +// }, +// "version": "1.0.1" +// } +// ``` +// The object provides metadata about the API. The metadata MAY be used by the clients if needed, and MAY be presented in editing or documentation generation tools for convenience. +pub struct Info { +pub mut: + title string @[required] // The title of the API + summary string // A short summary of the API. + description string // A description of the API. CommonMark syntax MAY be used for rich text representation. + terms_of_service string // A URL to the Terms of Service for the API. This MUST be in the form of a URL. + contact Contact // The contact information for the exposed API. + license License // The license information for the exposed API. + version string @[required] // The version of the OpenAPI document (which is distinct from the OpenAPI Specification version or the API implementation version). +} + +// ```{ +// "name": "API Support", +// "url": "https://www.example.com/support", +// "email": "support@example.com" +// }``` +// Contact information for the exposed API. +pub struct Contact { +pub: + name string // The identifying name of the contact person/organization. + url string // The URL pointing to the contact information. This MUST be in the form of a URL. + email string // The email address of the contact person/organization. This MUST be in the form of an email address. +} + +// ```{ +// "name": "Apache 2.0", +// "identifier": "Apache-2.0" +// }``` +// License information for the exposed API. +pub struct License { +pub: + name string // The license name used for the API. + identifier string // An SPDX license expression for the API. The identifier field is mutually exclusive of the url field. + url string // A URL to the license used for the API. This MUST be in the form of a URL. The url field is mutually exclusive of the identifier field. +} + + +// ```{ +// "url": "https://development.gigantic-server.com/v1", +// "description": "Development server" +// }``` +pub struct ServerSpec { +pub: + url string @[required] // A URL to the target host. This URL supports ServerSpec Variables and MAY be relative, to indicate that the host location is relative to the location where the OpenAPI document is being served. Variable substitutions will be made when a variable is named in {brackets}. + description string // An optional string describing the host designated by the URL. CommonMark syntax MAY be used for rich text representation. + variables map[string]ServerVariable // A map between a variable name and its value. The value is used for substitution in the server’s URL template. +} + +// An object representing a ServerSpec Variable for server URL template substitution. +pub struct ServerVariable { +pub: + enum_ []string @[json: 'enum'] // An enumeration of string values to be used if the substitution options are from a limited set. + default_ string @[json: 'default'; required] // The default value to use for substitution, which SHALL be sent if an alternate value is not supplied. Note this behavior is different than the Schema Object’s treatment of default values, because in those cases parameter values are optional. + description string @[omitempty] // An optional description for the server variable. GitHub Flavored Markdown syntax MAY be used for rich text representation. +} + +pub struct Path {} + +// pub struct Reference { +// ref string @[json: 'ref'] // The reference identifier. This MUST be in the form of a URI. +// summary string // A short summary which by default SHOULD override that of the referenced component. If the referenced object-type does not allow a summary field, then this field has no effect. +// description string // A description which by default SHOULD override that of the referenced component. CommonMark syntax MAY be used for rich text representation. If the referenced object-type does not allow a description field, then this field has no effect. +// } + +pub type PathRef = Path | Reference + +pub struct Components { +pub mut: + schemas map[string]SchemaRef // An object to hold reusable Schema Objects. + responses map[string]ResponseRef // An object to hold reusable ResponseSpec Objects. + parameters map[string]ParameterRef // An object to hold reusable Parameter Objects. + examples map[string]ExampleRef // An object to hold reusable Example Objects. + request_bodies map[string]RequestBodyRef // An object to hold reusable Request Body Objects. + headers map[string]HeaderRef // An object to hold reusable Header Objects. + security_schemes map[string]SecuritySchemeRef // An object to hold reusable Security Scheme Objects. + links map[string]LinkRef // An object to hold reusable Link Objects. + callbacks map[string]CallbackRef // An object to hold reusable Callback Objects. + path_items map[string]PathItemRef // An object to hold reusable Path Item Object. +} + + +type Items = SchemaRef | []SchemaRef + +// type Number = int + +// pub struct Request { +// description string // A brief description of the request body. This could contain examples of use. CommonMark syntax MAY be used for rich text representation. +// content map[string]MediaType // The content of the request body. The key is a media type (e.g., `application/json`) and the value describes it. +// required bool = false // Determines if the request body is required in the request. Defaults to false. +// } + +pub type ResponseRef = Reference | ResponseSpec +pub type ParameterRef = Parameter | Reference +pub type SecuritySchemeRef = Reference | SecurityScheme +pub type ExampleRef = Example | Reference +pub type RequestBodyRef = Reference | RequestBody +pub type HeaderRef = Header | Reference +pub type LinkRef = Link | Reference +pub type CallbackRef = Callback | Reference +pub type PathItemRef = PathItem | Reference +// type RequestRef = Reference | Request + +pub struct PathItem { +pub mut: + ref string @[omitempty] // Allows for a referenced definition of this path item. The referenced structure MUST be in the form of a Path Item Object. In case a Path Item Object field appears both in the defined object and the referenced object, the behavior is undefined. See the rules for resolving Relative References. + summary string @[omitempty] // An optional, string summary, intended to apply to all operations in this path. + description string @[omitempty] // An optional, string description, intended to apply to all operations in this path. CommonMark syntax MAY be used for rich text representation. + get Operation @[omitempty] // A definition of a GET operation on this path. + put Operation @[omitempty] // A definition of a PUT operation on this path. + post Operation @[omitempty] // A definition of a POST operation on this path. + delete Operation @[omitempty] // A definition of a DELETE operation on this path. + options Operation @[omitempty] // A definition of a OPTIONS operation on this path. + head Operation @[omitempty] // A definition of a HEAD operation on this path. + patch Operation @[omitempty] // A definition of a PATCH operation on this path. + trace Operation @[omitempty] // A definition of a TRACE operation on this path. + servers []ServerSpec @[omitempty] // An alternative server array to service all operations in this path. + parameters []Parameter @[omitempty]// A list of parameters that are applicable for all the operations described under this path. These parameters can be overridden at the operation level, but cannot be removed there. The list MUST NOT include duplicated parameters. A unique parameter is defined by a combination of a name and location. The list can use the Reference Object to link to parameters that are defined at the OpenAPI Object’s components/parameters. +} + +pub struct Operation { +pub mut: + tags []string @[omitempty]// A list of tags for API documentation control. Tags can be used for logical grouping of operations by resources or any other qualifier. + summary string @[omitempty]// A short summary of what the operation does. + description string @[omitempty] // A verbose explanation of the operation behavior. CommonMark syntax MAY be used for rich text representation. + external_docs ExternalDocumentation @[json: 'externalDocs'; omitempty] // Additional external documentation for this operation. + operation_id string @[json: 'operationId'; omitempty] // Unique string used to identify the operation. The id MUST be unique among all operations described in the API. The operationId value is case-sensitive. Tools and libraries MAY use the operationId to uniquely identify an operation, therefore, it is RECOMMENDED to follow common programming naming conventions. + parameters []Parameter @[omitempty]// A list of parameters that are applicable for this operation. If a parameter is already defined at the Path Item, the new definition will override it but can never remove it. The list MUST NOT include duplicated parameters. A unique parameter is defined by a combination of a name and location. The list can use the Reference Object to link to parameters that are defined at the OpenAPI Object’s components/parameters. + request_body RequestBodyRef @[json: 'requestBody'; omitempty] // The request body applicable for this operation. The requestBody is fully supported in HTTP methods where the HTTP 1.1 specification [RFC7231] has explicitly defined semantics for request bodies. In other cases where the HTTP spec is vague (such as GET, HEAD and DELETE), requestBody is permitted but does not have well-defined semantics and SHOULD be avoided if possible. + responses map[string]ResponseSpec @[omitempty] // The list of possible responses as they are returned from executing this operation. + callbacks map[string]CallbackRef @[omitempty] // A map of possible out-of band callbacks related to the parent operation. The key is a unique identifier for the Callback Object. Each value in the map is a Callback Object that describes a request that may be initiated by the API provider and the expected responses. + deprecated bool @[omitempty]// Declares this operation to be deprecated. Consumers SHOULD refrain from usage of the declared operation. Default value is false. + security []SecurityRequirement @[omitempty] // A declaration of which security mechanisms can be used for this operation. The list of values includes alternative security requirement objects that can be used. Only one of the security requirement objects need to be satisfied to authorize a request. To make security optional, an empty security requirement ({}) can be included in the array. This definition overrides any declared top-level security. To remove a top-level security declaration, an empty array can be used. + servers []ServerSpec @[omitempty]// An alternative server array to service this operation. If an alternative server object is specified at the Path Item Object or Root level, it will be overridden by this value. +} + +// TODO: currently using map[string]ResponseSpec +pub struct Responses { +pub: + default ResponseRef +} + +pub struct Callback { +pub: + callback string +} + +pub struct Link { +pub: + link string +} + +pub struct Header { +pub: + header string +} + +pub struct ResponseSpec { +pub mut: + description string @[required] // A description of the response. CommonMark syntax MAY be used for rich text representation. + headers map[string]HeaderRef // Maps a header name to its definition. [RFC7230] states header names are case insensitive. If a response header is defined with the name "Content-Type", it SHALL be ignored. + content map[string]MediaType // A map containing descriptions of potential response payloads. The key is a media type or media type range and the value describes it. For responses that match multiple keys, only the most specific key is applicable. e.g. text/plain overrides text/* + links map[string]LinkRef // A map of operations links that can be followed from the response. The key of the map is a short name for the link, following the naming constraints of the names for Component Objects. +} + +// TODO: media type example any field +pub struct MediaType { +pub mut: + schema SchemaRef // The schema defining the content of the request, response, or parameter. + example string // Example of the media type. The example object SHOULD be in the correct format as specified by the media type. The example field is mutually exclusive of the examples field. Furthermore, if referencing a schema which contains an example, the example value SHALL override the example provided by the schema. + examples map[string]ExampleRef // Examples of the media type. Each example object SHOULD match the media type and specified schema if present. The examples field is mutually exclusive of the example field. Furthermore, if referencing a schema which contains an example, the examples value SHALL override the example provided by the schema. + encoding map[string]Encoding // A map between a property name and its encoding information. The key, being the property name, MUST exist in the schema as a property. The encoding object SHALL only apply to requestBody objects when the media type is multipart or application/x-www-form-urlencoded. +} + +pub struct Encoding { +pub: + content_type string @[json: 'contentType'] // The Content-Type for encoding a specific property. Default value depends on the property type: for object - application/json; for array – the default is defined based on the inner type; for all other cases the default is application/octet-stream. The value can be a specific media type (e.g. application/json), a wildcard media type (e.g. image/*), or a comma-separated list of the two types. + headers map[string]HeaderRef // A map allowing additional information to be provided as headers, for example Content-Disposition. Content-Type is described separately and SHALL be ignored in this section. This property SHALL be ignored if the request body media type is not a multipart. + style string // Describes how a specific property value will be serialized depending on its type. See Parameter Object for details on the style property. The behavior follows the same values as query parameters, including default values. This property SHALL be ignored if the request body media type is not application/x-www-form-urlencoded or multipart/form-data. If a value is explicitly defined, then the value of contentType (implicit or explicit) SHALL be ignored. + explode bool // When this is true, property values of type array or object generate separate parameters for each value of the array, or key-value-pair of the map. For other types of properties this property has no effect. When style is form, the default value is true. For all other styles, the default value is false. This property SHALL be ignored if the request body media type is not application/x-www-form-urlencoded or multipart/form-data. If a value is explicitly defined, then the value of contentType (implicit or explicit) SHALL be ignored. + allow_reserved bool // Determines whether the parameter value SHOULD allow reserved characters, as defined by [RFC3986] :/?#[]@!$&'()*+,;= to be included without percent-encoding. The default value is false. This property SHALL be ignored if the request body media type is not application/x-www-form-urlencoded or multipart/form-data. If a value is explicitly defined, then the value of contentType (implicit or explicit) SHALL be ignored. +} + +pub struct Parameter { +pub mut: + name string @[required] // The name of the parameter. Parameter names are case sensitive. + in_ string @[json: 'in'; required] // The location of the parameter. Possible values are "query", "header", "path" or "cookie". + description string @[omitempty]// A brief description of the parameter. This could contain examples of use. CommonMark syntax MAY be used for rich text representation. + required bool @[omitempty]// Determines whether this parameter is mandatory. If the parameter location is "path", this property is REQUIRED and its value MUST be true. Otherwise, the property MAY be included and its default value is false. + deprecated bool @[omitempty]// Specifies that a parameter is deprecated and SHOULD be transitioned out of usage. Default value is false. + allow_empty_value bool @[json: 'allowEmptyValue'] // Sets the ability to pass empty-valued parameters. This is valid only for query parameters and allows sending a parameter with an empty value. Default value is false. If style is used, and if behavior is n/a (cannot be serialized), the value of allowEmptyValue SHALL be ignored. Use of this property is NOT RECOMMENDED, as it is likely to be removed in a later revision. + schema SchemaRef // The schema defining the type used for the parameter. +} + +pub struct Example { + example string +} + +pub struct SecurityScheme {} + +pub struct RequestBody { +pub mut: + description string // A brief description of the request body. This could contain examples of use. CommonMark syntax MAY be used for rich text representation. + content map[string]MediaType // The content of the request body. The key is a media type (e.g., `application/json`) and the value describes it. + required bool // Determines if the request body is required in the request. Defaults to false. +} + +pub struct SecurityRequirement {} + +pub struct Tag {} + +pub struct ExternalDocumentation { + external string +} diff --git a/crystallib/web/openapi/model_test.v b/crystallib/web/openapi/model_test.v new file mode 100644 index 000000000..73b96dc56 --- /dev/null +++ b/crystallib/web/openapi/model_test.v @@ -0,0 +1,400 @@ +module openapi + +import os +import json +import freeflowuniverse.crystallib.data.jsonschema {Schema, Reference, SchemaRef} + +const spec_path = '${os.dir(@FILE)}/testdata/openapi.json' +const spec_json = os.read_file(spec_path) or {panic(err)} + +const spec = openapi.OpenAPI{ + openapi: '3.0.3' + info: openapi.Info{ + title: 'Pet Store API' + description: 'A sample API for a pet store' + version: '1.0.0' + } + servers: [ + openapi.Server{ + url: 'https://api.petstore.example.com/v1' + description: 'Production server' + }, + openapi.Server{ + url: 'https://staging.petstore.example.com/v1' + description: 'Staging server' + } + ] + paths: { + '/pets': openapi.PathItem{ + get: openapi.Operation{ + summary: 'List all pets' + operation_id: 'listPets' + parameters: [ + openapi.Parameter{ + name: 'limit' + in_: 'query' + description: 'Maximum number of pets to return' + required: false + schema: Schema{ + typ: 'integer' + format: 'int32' + } + } + ] + responses: { + '200': openapi.Response{ + description: 'A paginated list of pets' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Pets' + } + } + } + } + '400': openapi.Response{ + description: 'Invalid request' + } + } + } + post: openapi.Operation{ + summary: 'Create a new pet' + operation_id: 'createPet' + request_body: openapi.RequestBody{ + required: true + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/NewPet' + } + } + } + } + responses: { + '201': openapi.Response{ + description: 'Pet created' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Pet' + } + } + } + } + '400': openapi.Response{ + description: 'Invalid input' + } + } + } + } + '/pets/{petId}': openapi.PathItem{ + get: openapi.Operation{ + summary: 'Get a pet by ID' + operation_id: 'getPet' + parameters: [ + openapi.Parameter{ + name: 'petId' + in_: 'path' + description: 'ID of the pet to retrieve' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '200': openapi.Response{ + description: 'A pet' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Pet' + } + } + } + } + '404': openapi.Response{ + description: 'Pet not found' + } + } + } + delete: openapi.Operation{ + summary: 'Delete a pet by ID' + operation_id: 'deletePet' + parameters: [ + openapi.Parameter{ + name: 'petId' + in_: 'path' + description: 'ID of the pet to delete' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '204': openapi.Response{ + description: 'Pet deleted' + } + '404': openapi.Response{ + description: 'Pet not found' + } + } + } + } + '/orders': openapi.PathItem{ + get: openapi.Operation{ + summary: 'List all orders' + operation_id: 'listOrders' + responses: { + '200': openapi.Response{ + description: 'A list of orders' + content: { + 'application/json': openapi.MediaType{ + schema: Schema{ + typ: 'array' + items: SchemaRef(Reference{ + ref: '#/components/schemas/Order' + }) + } + } + } + } + } + } + } + '/orders/{orderId}': openapi.PathItem{ + get: openapi.Operation{ + summary: 'Get an order by ID' + operation_id: 'getOrder' + parameters: [ + openapi.Parameter{ + name: 'orderId' + in_: 'path' + description: 'ID of the order to retrieve' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '200': openapi.Response{ + description: 'An order' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/Order' + } + } + } + } + '404': openapi.Response{ + description: 'Order not found' + } + } + } + delete: openapi.Operation{ + summary: 'Delete an order by ID' + operation_id: 'deleteOrder' + parameters: [ + openapi.Parameter{ + name: 'orderId' + in_: 'path' + description: 'ID of the order to delete' + required: true + schema: Schema{ + typ: 'integer' + format: 'int64' + } + } + ] + responses: { + '204': openapi.Response{ + description: 'Order deleted' + } + '404': openapi.Response{ + description: 'Order not found' + } + } + } + } + '/users': openapi.PathItem{ + post: openapi.Operation{ + summary: 'Create a user' + operation_id: 'createUser' + request_body: openapi.RequestBody{ + required: true + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/NewUser' + } + } + } + } + responses: { + '201': openapi.Response{ + description: 'User created' + content: { + 'application/json': openapi.MediaType{ + schema: Reference{ + ref: '#/components/schemas/User' + } + } + } + } + } + } + } + } + components: openapi.Components{ + schemas: { + 'Pet': SchemaRef(Schema{ + typ: 'object' + required: ['id', 'name'] + properties: { + 'id': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'name': SchemaRef(Schema{ + typ: 'string' + }) + 'tag': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + 'NewPet': SchemaRef(Schema{ + typ: 'object' + required: ['name'] + properties: { + 'name': SchemaRef(Schema{ + typ: 'string' + }) + 'tag': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + 'Pets': SchemaRef(Schema{ + typ: 'array' + items: SchemaRef(Reference{ + ref: '#/components/schemas/Pet' + }) + }) + 'Order': SchemaRef(Schema{ + typ: 'object' + required: ['id', 'petId', 'quantity', 'shipDate'] + properties: { + 'id': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'petId': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'quantity': SchemaRef(Schema{ + typ: 'integer' + format: 'int32' + }) + 'shipDate': SchemaRef(Schema{ + typ: 'string' + format: 'date-time' + }) + 'status': SchemaRef(Schema{ + typ: 'string' + enum_: ['placed', 'approved', 'delivered'] + }) + 'complete': SchemaRef(Schema{ + typ: 'boolean' + }) + } + }) + 'User': SchemaRef(Schema{ + typ: 'object' + required: ['id', 'username'] + properties: { + 'id': SchemaRef(Schema{ + typ: 'integer' + format: 'int64' + }) + 'username': SchemaRef(Schema{ + typ: 'string' + }) + 'email': SchemaRef(Schema{ + typ: 'string' + }) + 'phone': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + 'NewUser': SchemaRef(Schema{ + typ: 'object' + required: ['username'] + properties: { + 'username': SchemaRef(Schema{ + typ: 'string' + }) + 'email': SchemaRef(Schema{ + typ: 'string' + }) + 'phone': SchemaRef(Schema{ + typ: 'string' + }) + } + }) + } +} +} + +pub fn testsuite_begin() {} + +fn test_decode() { + decoded := json_decode(spec_json)! + + assert decoded.openapi == spec.openapi + assert decoded.info == spec.info + assert decoded.servers == spec.servers + for key, path in decoded.paths { + assert path.ref == spec.paths[key].ref, 'Paths ${key} dont match.' + assert path.summary == spec.paths[key].summary, 'Paths ${key} dont match.' + assert path.description == spec.paths[key].description, 'Paths ${key} dont match.' + match_operations(path.get, spec.paths[key].get) + match_operations(path.put, spec.paths[key].put) + match_operations(path.post, spec.paths[key].post) + match_operations(path.delete, spec.paths[key].delete) + } + assert decoded.webhooks == spec.webhooks + for key, schema in decoded.components.schemas { + assert schema == spec.components.schemas[key], 'Schemas ${key} dont match.' + } + assert decoded.components == spec.components + assert decoded.security == spec.security +} + +fn test_encode() { + spec.json_encode() +} + +fn match_operations(a Operation, b Operation) { + println(a.responses['200'].content['application/json'].schema) + assert a.tags == b.tags, 'Tags do not match.' + assert a.summary == b.summary, 'Summary does not match.' + assert a.description == b.description, 'Description does not match.' + assert a.external_docs == b.external_docs, 'External documentation does not match.' + assert a.operation_id == b.operation_id, 'Operation ID does not match.' + assert a.parameters == b.parameters, 'Parameters do not match.' + assert a.request_body == b.request_body, 'Request body does not match.' + assert a.responses == b.responses, 'Responses do not match.' + assert a.callbacks == b.callbacks, 'Callbacks do not match.' + assert a.deprecated == b.deprecated, 'Deprecated flag does not match.' + assert a.security == b.security, 'Security requirements do not match.' + assert a.servers == b.servers, 'Servers do not match.' +} \ No newline at end of file diff --git a/crystallib/web/openapi/readme.md b/crystallib/web/openapi/readme.md new file mode 100644 index 000000000..e69de29bb diff --git a/crystallib/web/openapi/server.v b/crystallib/web/openapi/server.v new file mode 100644 index 000000000..08b0057da --- /dev/null +++ b/crystallib/web/openapi/server.v @@ -0,0 +1,217 @@ +module openapi + +import veb +import json +import freeflowuniverse.crystallib.data.jsonschema {Schema} +import x.json2 {Any} +import net.http + +pub struct Controller { +pub: + specification OpenAPI +pub mut: + handler IHandler +} + +pub struct Context { + veb.Context +} + +// Matches a request path against OpenAPI path templates in the parsed structs +// Returns the matching path key and corresponding PathItem if found +fn match_path(req_path string, spec OpenAPI) !PathItem { + // Iterate through all paths in the OpenAPI specification + for template, path_item in spec.paths { + if is_path_match(req_path, template) { + // Return the matching path template and its PathItem + return path_item + } + } + // If no match is found, return an error + return error('Path not found') +} + +// Checks if a request path matches a given OpenAPI path template +// Allows for dynamic path segments like `{petId}` in templates +fn is_path_match(req_path string, template string) bool { + // Split the request path and template into segments + req_segments := req_path.split('/') + template_segments := template.split('/') + + // If the number of segments doesn't match, the paths can't match + if req_segments.len != template_segments.len { + return false + } + + // Compare each segment in the template and request path + for i, segment in template_segments { + // If the segment is not dynamic (doesn't start with `{`), ensure it matches exactly + if !segment.starts_with('{') && segment != req_segments[i] { + return false + } + } + // If all segments match or dynamic segments are valid, return true + return true +} + +@['/:path...'; get; post; put; delete; patch] +pub fn (mut server Controller) index(mut ctx Context, path string) veb.Result { + println('Requested path: $path') + + // Extract the HTTP method + method := ctx.req.method.str().to_lower() + + // Matches the request path against the OpenAPI specification and retrieves the corresponding PathItem + path_item := match_path(path, server.specification) or { + // Return a 404 error if no matching path is found + return ctx.not_found() + } + + + // // Check if the path exists in the OpenAPI specification + // path_item := server.specification.paths[path] or { + // // Return a 404 error if the path is not defined + // return ctx.not_found() + // } + + // Match the HTTP method with the OpenAPI specification + operation := match method { + 'get' { path_item.get } + 'post' { path_item.post } + 'put' { path_item.put } + 'delete' { path_item.delete } + 'patch' { path_item.patch } + else { + // Return 405 Method Not Allowed if the method is not supported + return ctx.method_not_allowed() + } + } + + + mut arg_map := map[string]Any + path_arg := path.all_after_last('/') + // the OpenAPI Parameter specification belonging to the path argument + arg_params := operation.parameters.filter(it.in_ == 'path') + if arg_params.len > 1 { + // TODO: use path template to support multiple arguments (right now just last arg supported) + panic('implement') + } else if arg_params.len == 1 { + arg_map[arg_params[0].name] = arg_params[0].typed(path_arg) + } + + mut parameters := ctx.query.clone() + // Build the Request object + request := Request{ + path: path + operation: operation + method: method + arguments: arg_map + parameters: parameters + body: ctx.req.data + header: ctx.req.header + } + + // Use the handler to process the request + response := server.handler.handle(request) or { + // Use OpenAPI spec to determine the response status for the error + return ctx.handle_error(operation.responses, err) + } + + // Return the response to the client + ctx.res.set_status(response.status) + + // ctx.res.header = response.header + // ctx.set_content_type('application/json') + + // return ctx.ok('[]') + return ctx.send_response_to_client('application/json', response.body) +} + +// Handles errors and maps them to OpenAPI-defined response statuses +fn (mut ctx Context) handle_error(possible_responses map[string]ResponseSpec, err IError) veb.Result { + // Match the error with the defined responses + for code, _ in possible_responses { + if matches_error_to_status(err, code.int()) { + ctx.res.set_status(http.status_from_int(code.int())) + ctx.set_content_type('application/json') + return ctx.send_response_to_client( + 'application/json', + '{"error": "$err.msg()", "status": $code}' + ) + } + } + + // Default to 500 Internal Controller Error if no match is found + ctx.res.set_status(.internal_server_error) + ctx.set_content_type('application/json') + return ctx.send_response_to_client( + 'application/json', + '{"error": "Internal Controller Error", "status": 500}' + ) +} + +// Helper to match an error to a specific response status +fn matches_error_to_status(err IError, status int) bool { + // This can be customized to map specific errors to statuses + // For simplicity, we'll use a direct comparison here. + return err.code() == status +} + +// Helper for 405 Method Not Allowed response +fn (mut ctx Context) method_not_allowed() veb.Result { + ctx.res.set_status(.method_not_allowed) + ctx.set_content_type('application/json') + return ctx.send_response_to_client( + 'application/json', + '{"error": "Method Not Allowed", "status": 405}' + ) +} + + + +pub fn (param Parameter) typed(value string) Any { + param_schema := param.schema as Schema + param_type := param_schema.typ + param_format := param_schema.format + + // Convert parameter value to corresponding type + typ := match param_type { + 'integer' { + param_format + } + 'number' { + param_format + } + else { + param_type // Leave as param type for unknown types + } + } + return typed(value, typ) +} + +// typed gets a value that is string and a desired type, and returns the typed string in Any Type. +pub fn typed(value string, typ string) Any { + match typ { + 'int32' { + return value.int() // Convert to int + } + 'int64' { + return value.i64() // Convert to i64 + } + 'string' { + return value // Already a string + } + 'boolean' { + return value.bool() // Convert to bool + } + 'float' { + return value.f32() // Convert to float + } + 'double' { + return value.f64() // Convert to double + } + else { + return value.f64() // Leave as string for unknown types + } + } +} \ No newline at end of file diff --git a/crystallib/core/openapi/templates/client.vtemplate b/crystallib/web/openapi/templates/client.vtemplate similarity index 100% rename from crystallib/core/openapi/templates/client.vtemplate rename to crystallib/web/openapi/templates/client.vtemplate diff --git a/crystallib/core/openapi/templates/petstor.json b/crystallib/web/openapi/templates/petstor.json similarity index 100% rename from crystallib/core/openapi/templates/petstor.json rename to crystallib/web/openapi/templates/petstor.json diff --git a/crystallib/core/openapi/templates/qdrant.json b/crystallib/web/openapi/templates/qdrant.json similarity index 100% rename from crystallib/core/openapi/templates/qdrant.json rename to crystallib/web/openapi/templates/qdrant.json diff --git a/crystallib/web/openapi/testdata/openapi.json b/crystallib/web/openapi/testdata/openapi.json new file mode 100644 index 000000000..c5ab2d9d4 --- /dev/null +++ b/crystallib/web/openapi/testdata/openapi.json @@ -0,0 +1,346 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Pet Store API", + "description": "A sample API for a pet store", + "version": "1.0.0" + }, + "servers": [ + { + "url": "https://api.petstore.example.com/v1", + "description": "Production server" + }, + { + "url": "https://staging.petstore.example.com/v1", + "description": "Staging server" + } + ], + "paths": { + "/pets": { + "get": { + "summary": "List all pets", + "operationId": "listPets", + "parameters": [ + { + "name": "limit", + "in": "query", + "description": "Maximum number of pets to return", + "required": false, + "schema": { + "type": "integer", + "format": "int32" + } + } + ], + "responses": { + "200": { + "description": "A paginated list of pets", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Pets" + } + } + } + }, + "400": { + "description": "Invalid request" + } + } + }, + "post": { + "summary": "Create a new pet", + "operationId": "createPet", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewPet" + } + } + } + }, + "responses": { + "201": { + "description": "Pet created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Pet" + } + } + } + }, + "400": { + "description": "Invalid input" + } + } + } + }, + "/pets/{petId}": { + "get": { + "summary": "Get a pet by ID", + "operationId": "getPet", + "parameters": [ + { + "name": "petId", + "in": "path", + "description": "ID of the pet to retrieve", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "A pet", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Pet" + } + } + } + }, + "404": { + "description": "Pet not found" + } + } + }, + "delete": { + "summary": "Delete a pet by ID", + "operationId": "deletePet", + "parameters": [ + { + "name": "petId", + "in": "path", + "description": "ID of the pet to delete", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "204": { + "description": "Pet deleted" + }, + "404": { + "description": "Pet not found" + } + } + } + }, + "/orders": { + "get": { + "summary": "List all orders", + "operationId": "listOrders", + "responses": { + "200": { + "description": "A list of orders", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Order" + } + } + } + } + } + } + } + }, + "/orders/{orderId}": { + "get": { + "summary": "Get an order by ID", + "operationId": "getOrder", + "parameters": [ + { + "name": "orderId", + "in": "path", + "description": "ID of the order to retrieve", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "An order", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Order" + } + } + } + }, + "404": { + "description": "Order not found" + } + } + }, + "delete": { + "summary": "Delete an order by ID", + "operationId": "deleteOrder", + "parameters": [ + { + "name": "orderId", + "in": "path", + "description": "ID of the order to delete", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "204": { + "description": "Order deleted" + }, + "404": { + "description": "Order not found" + } + } + } + }, + "/users": { + "post": { + "summary": "Create a user", + "operationId": "createUser", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewUser" + } + } + } + }, + "responses": { + "201": { + "description": "User created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/User" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "Pet": { + "type": "object", + "required": ["id", "name"], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "name": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "NewPet": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "Pets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Pet" + } + }, + "Order": { + "type": "object", + "required": ["id", "petId", "quantity", "shipDate"], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "petId": { + "type": "integer", + "format": "int64" + }, + "quantity": { + "type": "integer", + "format": "int32" + }, + "shipDate": { + "type": "string", + "format": "date-time" + }, + "status": { + "type": "string", + "enum": ["placed", "approved", "delivered"] + }, + "complete": { + "type": "boolean" + } + } + }, + "User": { + "type": "object", + "required": ["id", "username"], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "username": { + "type": "string" + }, + "email": { + "type": "string" + }, + "phone": { + "type": "string" + } + } + }, + "NewUser": { + "type": "object", + "required": ["username"], + "properties": { + "username": { + "type": "string" + }, + "email": { + "type": "string" + }, + "phone": { + "type": "string" + } + } + } + } + } +} \ No newline at end of file diff --git a/crystallib/webserver/livekit/meet/play.v b/crystallib/webserver/livekit/meet/play.v new file mode 100644 index 000000000..547b2eb83 --- /dev/null +++ b/crystallib/webserver/livekit/meet/play.v @@ -0,0 +1,26 @@ +module meet + +import freeflowuniverse.crystallib.core.playbook + +pub fn play(mut plbook playbook.PlayBook) !&App { + + livekit_actions := plbook.find(filter: 'livekit.')! + if livekit_actions.len == 0 { + return error('no livekit actions found') + } + + for action in livekit_actions { + mut p := action.params + + match action.name { + 'livekit.configure' { + config := action.params.decode[AppConfig]()! + return new(config) + } + else { + println('Unknown action: ${action.name}') + } + } + } + return error('no configuration action found for livekit') +} \ No newline at end of file diff --git a/examples/clients/meilisearch/meilisearch.vsh b/examples/clients/meilisearch/meilisearch.vsh index 572cb25dc..87eaa24b5 100755 --- a/examples/clients/meilisearch/meilisearch.vsh +++ b/examples/clients/meilisearch/meilisearch.vsh @@ -2,12 +2,6 @@ import freeflowuniverse.crystallib.clients.meilisearch - -factory := new_factory(host:'http://localhost:7700', api_key:'be61fdce-c5d4-44bc-886b-3a484ff6c531') -mut client := factory.get()! - - - struct MeiliDocument { pub mut: id int @@ -15,4 +9,7 @@ pub mut: content string } -//to complete for doc, geo & vector (AI) index \ No newline at end of file +mut client := meilisearch.get()! +version := client.version()! +println('version: ${version}') + diff --git a/examples/clients/stellar/accounts.vsh b/examples/clients/stellar/accounts.vsh new file mode 100755 index 000000000..ab6a3640c --- /dev/null +++ b/examples/clients/stellar/accounts.vsh @@ -0,0 +1,30 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.crystallib.blockchain.stellar + +// mut client := stellar.new_client( +// account_name:"default", +// account_secret: "SA5AH6PDCKPP4P7XNWX6W4SKVDS6C2GECH4BQDKPDG5JQC3GFRGAB377", +// network: .testnet +// )! + +println('Creating a new account 1') +account1 := stellar.generate_keys(name: 'account1', network: .testnet)! +println('Account 1: ${account1}') + +println('Creating a new account 2') +// Use generate account on testnet with fund=true to add the trustline +account2 := stellar.generate_keys(name: 'account2', network: .testnet, fund: true)! +println('Account 2: ${account2}') + +mut client := stellar.new_client( + account_name: account2.name + account_secret: account2.secret + network: .testnet + cache: false +)! + +// Use this method to add the trustline to the account +tx := client.create_account(address: account1.address, starting_balance: 10000000)! + +println('tx: ${tx}') diff --git a/examples/clients/stellar/load_money.vsh b/examples/clients/stellar/archive/load_money.vsh similarity index 86% rename from examples/clients/stellar/load_money.vsh rename to examples/clients/stellar/archive/load_money.vsh index 154b6eea7..75e081d13 100755 --- a/examples/clients/stellar/load_money.vsh +++ b/examples/clients/stellar/archive/load_money.vsh @@ -9,7 +9,7 @@ import freeflowuniverse.crystallib.core.texttools import freeflowuniverse.crystallib.blockchain -heropath := '/Users/despiegk1/private_new/tft/data/hero' +heropath := '~/private_new/tft/data/hero' mut bc:=blockchain.get()! diff --git a/examples/clients/stellar/load_toml_accounts.vsh b/examples/clients/stellar/archive/load_toml_accounts.vsh similarity index 100% rename from examples/clients/stellar/load_toml_accounts.vsh rename to examples/clients/stellar/archive/load_toml_accounts.vsh diff --git a/examples/clients/stellar/stellar.vsh b/examples/clients/stellar/archive/stellar.vsh similarity index 54% rename from examples/clients/stellar/stellar.vsh rename to examples/clients/stellar/archive/stellar.vsh index 22316d3ff..6c4c42a93 100755 --- a/examples/clients/stellar/stellar.vsh +++ b/examples/clients/stellar/archive/stellar.vsh @@ -1,22 +1,22 @@ #!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + import freeflowuniverse.crystallib.clients.stellar -mut cl:=b2.get(instance:"default")! +mut cl := b2.get(instance: 'default')! // cl.config_delete()! -mut cfg:=cl.config()! -if cfg.appkey==""{ +mut cfg := cl.config()! +if cfg.appkey == '' { // will ask questions if not filled in yet // cl.config_interactive()! } println(cfg) -//will now change programatically -cfg.description="something else" -//will now save the config +// will now change programatically +cfg.description = 'something else' +// will now save the config cl.config_save()! -//we will now see how the description has been overwritten +// we will now see how the description has been overwritten println(cfg) - diff --git a/examples/clients/stellar/example.vsh b/examples/clients/stellar/example.vsh deleted file mode 100755 index 9158fb1a0..000000000 --- a/examples/clients/stellar/example.vsh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run - -import freeflowuniverse.crystallib.blockchain.stellar - -mut client := stellar.new_stellar_client(network: 'testnet', default_account: 'mario')! - - -client.add_keys(secret: 'SAAK2IJ5VNN453BQMDBR3WIL4TSBATIAW6I5QGKPUQZ6YBRON2HXU7N2')! -client.add_signer(address: 'GAAEAXBM2BTW4SK6Z4OZRWVNH4KM5PUF7RR746EBEWH5NMSUZTU6U7AK')! -client.remove_signer(address: 'GAAEAXBM2BTW4SK6Z4OZRWVNH4KM5PUF7RR746EBEWH5NMSUZTU6U7AK')! -client.merge_accounts(address: 'GAAEAXBM2BTW4SK6Z4OZRWVNH4KM5PUF7RR746EBEWH5NMSUZTU6U7AK')! \ No newline at end of file diff --git a/examples/clients/stellar/horizon.vsh b/examples/clients/stellar/horizon.vsh index 65d5d608c..34f39c3bd 100755 --- a/examples/clients/stellar/horizon.vsh +++ b/examples/clients/stellar/horizon.vsh @@ -1,16 +1,24 @@ #!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run -import toml -import toml.to -import json -import os -import freeflowuniverse.crystallib.data.encoderhero -import freeflowuniverse.crystallib.core.texttools import freeflowuniverse.crystallib.blockchain.stellar +// create and fund a new account on testnet +generated_account := stellar.generate_keys(name: 'account', network: .testnet, fund: true)! +println('Account: ${generated_account}') -mut cl:= stellar.new_horizon_client()! +mut stellar_client := stellar.new_client( + account_name: generated_account.name + account_secret: generated_account.secret + network: .testnet + cache: false +)! -mut a:= cl.get_account("GB2KXHBMYRIKWNQCDK7TCOQ6ANOCD2OFTBHSS5FNIN7OS67I2UBMRHCO")! +mut horizon_client := stellar.new_horizon_client(.testnet)! -println(a) \ No newline at end of file +// get account information +mut account := horizon_client.get_account(generated_account.address)! +println('account: ${account}') + +// get infromation about last transaction for this account +last_tx := horizon_client.get_last_transaction(generated_account.address)! +println('last tx: ${last_tx}') diff --git a/examples/clients/stellar/multisign.vsh b/examples/clients/stellar/multisign.vsh deleted file mode 100755 index a6f6b7154..000000000 --- a/examples/clients/stellar/multisign.vsh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run - -import toml -import toml.to -import json -import os -import freeflowuniverse.crystallib.data.encoderhero -import freeflowuniverse.crystallib.core.texttools -import freeflowuniverse.crystallib.blockchain.stellar - - - -//make the required accou ts - - -for x in ["mother","signer1","signer2","dest"] { - - if cl.account_exists(x) { - println("Account $x exists") - } else { - println("Account $x does not exist") - mut cl:= stellar.new_stellar_client()! - } - if cl.account_funded(x) > 10 { - println("Account $x exists and is enough funded") - } else { - println("Account $x is not funded") - fundingamount:=cl.account_fund(x)! - assert fundingamount>0 - } -} - -cl.signers_add(name:'mother',pubkeys:["signer1","signer2"]) ! - -//TODO: now check if we can again add signers, even if some already existed - -//TODO: now show how we can send money from - -cl.send(name:'mother',dest:'dest',amount:1, asset:'xlm') ! - -cl.sign(... not sure how to sign one bu one, is what our users will have to do) ! \ No newline at end of file diff --git a/examples/clients/stellar/payments.vsh b/examples/clients/stellar/payments.vsh new file mode 100755 index 000000000..7a36a565c --- /dev/null +++ b/examples/clients/stellar/payments.vsh @@ -0,0 +1,57 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.crystallib.blockchain.stellar + +account1 := stellar.generate_keys(name: 'account1', network: .testnet, fund: true)! +println('Account 1: ${account1}') + +account2 := stellar.generate_keys(name: 'account2', network: .testnet, fund: true)! +println('Account 2: ${account2}') + +account3 := stellar.generate_keys(name: 'account3', network: .testnet, fund: true)! +println('Account 3: ${account3}') + +mut client := stellar.new_client( + account_name: 'default' + account_secret: account1.secret + network: .testnet + cache: false +)! + +mut signers := [ + account2.secret, + account3.secret, +] + +// every operation belongs to one of the thresholds (low, med, high) +// a payment oepration uses the med threshold +mut hash := client.update_threshold(med_threshold: 5)! +println('update threshold tx hash: ${hash}') + +signer1 := stellar.new_signer( + key: account1.address + weight: 3 +) + +signer2 := stellar.new_signer( + key: account2.address + weight: 4 +) + +signer3 := stellar.new_signer( + key: account3.address + weight: 5 +) + +mut hash2 := client.add_signers( + signers_to_add: [signer1, signer2, signer3] +)! +println('add signer tx hash: ${hash2}') + +// this would fail if we don't add enough singers +hash2 = client.payment_send( + destination: account2.address + amount: 200 + signers: signers +)! +println('payment tx hash: ${hash2}') diff --git a/examples/clients/stellar/readme.md b/examples/clients/stellar/readme.md new file mode 100644 index 000000000..0ee3add2f --- /dev/null +++ b/examples/clients/stellar/readme.md @@ -0,0 +1,3 @@ +# Stellar Examples + + diff --git a/examples/clients/stellar/signers.vsh b/examples/clients/stellar/signers.vsh new file mode 100755 index 000000000..abd3e48cb --- /dev/null +++ b/examples/clients/stellar/signers.vsh @@ -0,0 +1,49 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.crystallib.blockchain.stellar + +account1 := stellar.generate_keys(name: 'account1', network: .testnet, fund: true)! +println('Account 1: ${account1}') + +account2 := stellar.generate_keys(name: 'account2', network: .testnet, fund: true)! +println('Account 2: ${account2}') + +account3 := stellar.generate_keys(name: 'account3', network: .testnet, fund: true)! +println('Account 3: ${account3}') + +mut client := stellar.new_client( + account_name: 'default' + account_secret: account1.secret + network: .testnet + cache: false +)! + +// If you have a saved keys you can use the get_client method without specifying the account_secret. +// mut client := stellar.get_client(account_name:"default", network: .testnet)! + +signer1 := stellar.new_signer( + key: account1.address + weight: 3 +) + +signer2 := stellar.new_signer( + key: account2.address + weight: 4 +) + +signer3 := stellar.new_signer( + key: account3.address + weight: 5 +) + +mut hash := client.add_signers( + signers_to_add: [signer1, signer2, signer3] // signers to add to this account + signers: [account1.secret] // signers that may sign this transaction +)! +println('add signer tx hash: ${hash}') + +hash2 := client.remove_signer( + address: account2.address // signer to remove + signers: [account3.secret] // signers that may sign this transaction +)! +println('remove signer tx hash: ${hash2}') diff --git a/examples/clients/stellar/trading.vsh b/examples/clients/stellar/trading.vsh new file mode 100755 index 000000000..6d125d4cb --- /dev/null +++ b/examples/clients/stellar/trading.vsh @@ -0,0 +1,59 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.crystallib.blockchain.stellar + +tft_issuer := 'GCGE3IQWC4QIOJ7WVLIHZMXSE623CMXWQVMK4JWSRX2V3TXZEW3RHDR6' + +account1 := stellar.generate_keys(name: 'account1', network: .testnet, fund: true)! +println('Account 1: ${account1}') + +mut client := stellar.new_client( + account_name: 'default' + account_secret: account1.secret + network: .testnet + cache: false +)! + +mut hash := client.add_trust_line( + asset_code: 'TFT' + issuer: tft_issuer // tft issuer id +)! +println('add tft trustline tx hash: ${hash}') + +// Make sell offer +sell_offer_args := stellar.OfferArgs{ + selling: stellar.OfferAssetType('native') + buying: stellar.OfferAssetType(stellar.new_asset_type('TFT', tft_issuer)) + sell: true + amount: 50 + price: 10 +} +mut sell_offer_result := client.create_offer(sell_offer_args)! +if sell_offer_result.claimed { + println('Offer created and claimed by ${sell_offer_result.offer_id}') +} else { + println('Offer ${sell_offer_result.offer_id} is created') +} + +// Make buy offer +mut buy_offer_args := stellar.OfferArgs{ + selling: stellar.OfferAssetType('native') + buying: stellar.OfferAssetType(stellar.new_asset_type('TFT', tft_issuer)) + buy: true + amount: 50 + price: 10 +} + +mut buy_offer_result := client.create_offer(buy_offer_args)! +if buy_offer_result.claimed { + println('Offer created and claimed by ${buy_offer_result.offer_id}') +} else { + println('Offer ${buy_offer_result.offer_id} is created') +} + +buy_offer_args.amount = 100 +client.update_offer(buy_offer_id, buy_offer_args)! +println('offer ${buy_offer_id} is update') + +client.delete_offer(sell_offer_id, sell_offer_args)! +println('sell offer ${sell_offer_id} is deleted') diff --git a/examples/clients/stellar/trading_bot.vsh b/examples/clients/stellar/trading_bot.vsh new file mode 100755 index 000000000..30ad8f83d --- /dev/null +++ b/examples/clients/stellar/trading_bot.vsh @@ -0,0 +1,23 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.crystallib.blockchain.stellar.tradingbot +import freeflowuniverse.crystallib.blockchain.stellar + +// account1 := stellar.generate_keys(name: 'account1', network: .testnet, fund: true)! +// println('Account 1: ${account1}') + +tft_issuer := 'GCGE3IQWC4QIOJ7WVLIHZMXSE623CMXWQVMK4JWSRX2V3TXZEW3RHDR6' + +mut bot := tradingbot.new( + account_secret: 'SDKKNNX5NSYR62BUMIAZM6JDIGCUHYLWOHLM7NWICPVCOEIBF544TGM2' + buying_asset_type: 'native' + selling_asset_code: 'TFT' + selling_asset_issuer: tft_issuer + selling_target_price: 1000 + buying_target_price: 0.0001 + selling_amount: 10 + buying_amount: 10 + network: .testnet +)! + +bot.run()! diff --git a/examples/crystallib.code-workspace b/examples/crystallib.code-workspace index bec3812eb..d2ef42ae8 100644 --- a/examples/crystallib.code-workspace +++ b/examples/crystallib.code-workspace @@ -3,27 +3,18 @@ { "path": "../crystallib" }, - { - "path": "." - }, { "path": "../aiprompts" }, { "path": "../cli/hero" }, - { - "path": "../vscodeplugin" - }, { "path": "../scripts" }, { - "path": "../research" - }, - { - "path": "../tools" - }, + "path": "../examples" + }, { "path": "../cli" } diff --git a/examples/data/.gitignore b/examples/data/.gitignore new file mode 100644 index 000000000..257a899ac --- /dev/null +++ b/examples/data/.gitignore @@ -0,0 +1 @@ +doctree \ No newline at end of file diff --git a/examples/data/doctree.vsh b/examples/data/doctree.vsh deleted file mode 100755 index a5b0991ba..000000000 --- a/examples/data/doctree.vsh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run - -import freeflowuniverse.crystallib.data.doctree -import os - -testpath := '${os.home_dir()}/code/github/freeflowuniverse/crystallib/crystallib/data/doctree/testdata/collections/' - -mut tree := doctree.new()! - -tree.scan(path: testpath)! - -tree.process()! // process includes - -mut p := tree.page_get_processed('riverlov:introduction.md')! -// println(p) -// mut mydoc := p.doc()! -// println(mydoc) - -mut p2 := tree.page_get_processed('riverlov:aboutus.md')! -//mut mydoc2 := p2.doc()! -// // println(mydoc2.defpointers()) -// println(mydoc2) -println(p2.get_markdown()!) - -// tree.export(dest: '/tmp/remove')! diff --git a/examples/data/doctree/.gitignore b/examples/data/doctree/.gitignore new file mode 100644 index 000000000..be2d5fc56 --- /dev/null +++ b/examples/data/doctree/.gitignore @@ -0,0 +1 @@ +destination \ No newline at end of file diff --git a/examples/data/doctree/example_include.vsh b/examples/data/doctree/example_include.vsh new file mode 100755 index 000000000..eef89971b --- /dev/null +++ b/examples/data/doctree/example_include.vsh @@ -0,0 +1,26 @@ +#!/usr/bin/env -S v -enable-globals run + +import freeflowuniverse.crystallib.data.doctree +import os + +const test_dir = os.join_path(os.home_dir(), 'code/github/freeflowuniverse/crystallib/crystallib/data/doctree/testdata/process_includes_test') + +/* + 1- use 3 pages in testdata: + - page1 includes page2 + - page2 includes page3 + 2- create tree + 3- invoke process_includes + 4- check pages markdown +*/ +mut tree := doctree.new(name: 'example')! +tree.scan(path: test_dir)! +tree.process_includes()! + +mut page1 := tree.page_get('col1:page1.md')! +mut page2 := tree.page_get('col2:page2.md')! +mut page3 := tree.page_get('col2:page3.md')! + +assert page1.get_markdown() or {''} == 'page3 content' +assert page2.get_markdown() or {''} == 'page3 content' +assert page3.get_markdown() or {''} == 'page3 content' \ No newline at end of file diff --git a/examples/data/doctree/example_mdbook.vsh b/examples/data/doctree/example_mdbook.vsh new file mode 100755 index 000000000..098cd194e --- /dev/null +++ b/examples/data/doctree/example_mdbook.vsh @@ -0,0 +1,50 @@ +#!/usr/bin/env -S v -enable-globals run + +import freeflowuniverse.crystallib.data.doctree +import freeflowuniverse.crystallib.core.pathlib +import freeflowuniverse.crystallib.web.mdbook +import os + + +// directory of collections used in the example +const example_dir = os.join_path(os.home_dir(), 'code/github/freeflowuniverse/crystallib/crystallib/data/doctree/testdata/tree_test') + +// create tree and scan collection dir +mut tree := doctree.new(name: 'example_mdbook')! +tree.scan(path: example_dir)! + +// check collections scanned as expected +assert tree.collections.len == 2 +assert tree.collections.keys() == ['fruits', 'test_vegetables'] + +example_dest := '${os.dir(@FILE)}/destination' + +// export tree +tree.export(destination: '${example_dest}/tree', reset: true)! + +mut mdb := mdbook.get()! + +mut summary_path := pathlib.get_file(path: '${example_dest}/SUMMARY.md', create: true)! +summar_content := ' +- [Page number 1](fruits/apple.md) +- [fruit intro](fruits/intro.md) +- [rpc page](rpc/tfchain.md) +- [vegies](test_vegetables/tomato.md) +' +summary_path.write(summar_content)! + +// generate mdbook from summary and exported collections +mut b:=mdb.generate( + name: 'mdbook_example' + title: 'MDBook Example' + summary_path: summary_path.path + publish_path: '${example_dest}/publish' + build_path: '${example_dest}/build' + export: true + collections: [ + '${example_dest}/tree/fruits' + '${example_dest}/tree/test_vegetables' + ] +)! + +b.open()! \ No newline at end of file diff --git a/examples/data/doctree_include.vsh b/examples/data/doctree_include.vsh deleted file mode 100755 index d8eb0b10c..000000000 --- a/examples/data/doctree_include.vsh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run - -import freeflowuniverse.crystallib.data.doctree -import freeflowuniverse.crystallib.web.mdbook -import os - -collections_path := '${os.home_dir()}/code/github/freeflowuniverse/crystallib/crystallib/data/doctree/testdata/includetest/' - -mut tree := doctree.new(name: 'test')! -tree.scan( - path: collections_path - heal: false -)! - -assert tree.collections.len == 3 - -// println(tree.collections.keys()) -assert tree.collections.keys() == ['riverlov', 'server', 'sub2'] - -dest := '/tmp/mdbooktest' -tree.export(dest: '${dest}/tree', reset: true)! -mut mdb := mdbook.get(instance: 'mdbooktest')! - -// mut cfg := mdbooks.config()! -// cfg.path_build = buildroot -// cfg.path_publish = publishroot - -mut b:=mdb.generate( - doctree_path: '${dest}/tree' - name: 'includetest' - title: 'Incude Test' - summary_path: '${os.home_dir()}/code/github/freeflowuniverse/crystallib/crystallib/data/doctree/testdata/includetest/summary.md' - summary_url: '' // because path given - publish_path: '${dest}/publish' - build_path: '${dest}/build' -)! - -b.open()! \ No newline at end of file diff --git a/examples/develop/gittools/example3.vsh b/examples/develop/gittools/example3.vsh new file mode 100755 index 000000000..ae8bd5c85 --- /dev/null +++ b/examples/develop/gittools/example3.vsh @@ -0,0 +1,25 @@ +#!/usr/bin/env -S v -cg -enable-globals run + +import os +import freeflowuniverse.crystallib.develop.gittools +import freeflowuniverse.crystallib.develop.performance + +mut silent := false + +coderoot := if 'CODEROOT' in os.environ() { + os.environ()['CODEROOT'] +} else {os.join_path(os.home_dir(), 'code')} + +mut gs := gittools.get()! +if coderoot.len > 0 { + //is a hack for now + gs = gittools.new(coderoot: coderoot)! +} + +mypath := gs.do( + recursive: true + cmd: 'list' +)! + +timer := performance.new('gittools') +timer.timeline() \ No newline at end of file diff --git a/examples/hero/generator/actor_from_openrpc/example.vsh b/examples/hero/generator/actor_from_openrpc/example.vsh new file mode 100755 index 000000000..e1c490da4 --- /dev/null +++ b/examples/hero/generator/actor_from_openrpc/example.vsh @@ -0,0 +1,18 @@ +#!/usr/bin/env -S v -w -n -enable-globals run + +import freeflowuniverse.crystallib.hero.baobab.generator +import freeflowuniverse.crystallib.hero.baobab.specification +import freeflowuniverse.crystallib.rpc.openrpc +import os + +const example_dir = os.dir(@FILE) +const openrpc_spec_path = os.join_path(example_dir, 'openrpc.json') + +// the actor specification obtained from the OpenRPC Specification +openrpc_spec := openrpc.new(path: openrpc_spec_path)! +actor_spec := specification.from_openrpc(openrpc_spec)! + +generator.generate_actor_module( + actor_spec, + interfaces: [.openrpc] +)! \ No newline at end of file diff --git a/examples/hero/generator/actor_from_openrpc/openrpc.json b/examples/hero/generator/actor_from_openrpc/openrpc.json new file mode 100644 index 000000000..03e1e211e --- /dev/null +++ b/examples/hero/generator/actor_from_openrpc/openrpc.json @@ -0,0 +1,132 @@ +{ + "openrpc": "1.0.0", + "info": { + "title": "PetStore API", + "version": "1.0.0" + }, + "methods": [ + { + "name": "petstore_client.GetPets", + "description": "finds pets in the system that the user has access to by tags and within a limit", + "params": [ + { + "name": "tags", + "description": "tags to filter by", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "limit", + "description": "maximum number of results to return", + "schema": { + "type": "integer" + } + } + ], + "result": { + "name": "pet_list", + "description": "all pets from the system, that mathes the tags", + "schema": { + "$ref": "#\/components\/schemas\/Pet" + } + } + }, + { + "name": "petstore_client.CreatePet", + "description": "creates a new pet in the store. Duplicates are allowed.", + "params": [ + { + "name": "new_pet", + "description": "Pet to add to the store.", + "schema": { + "$ref": "#\/components\/schemas\/NewPet" + } + } + ], + "result": { + "name": "pet", + "description": "the newly created pet", + "schema": { + "$ref": "#\/components\/schemas\/Pet" + } + } + }, + { + "name": "petstore_client.GetPetById", + "description": "gets a pet based on a single ID, if the user has access to the pet", + "params": [ + { + "name": "id", + "description": "ID of pet to fetch", + "schema": { + "type": "integer" + } + } + ], + "result": { + "name": "pet", + "description": "pet response", + "schema": { + "$ref": "#\/components\/schemas\/Pet" + } + } + }, + { + "name": "petstore_client.DeletePetById", + "description": "deletes a single pet based on the ID supplied", + "params": [ + { + "name": "id", + "description": "ID of pet to delete", + "schema": { + "type": "integer" + } + } + ], + "result": { + "name": "pet", + "description": "pet deleted", + "schema": { + "type": "null" + } + } + } + ], + "components": { + "schemas": { + "NewPet": { + "title": "NewPet", + "properties": { + "name": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "Pet": { + "title": "Pet", + "description": "a pet struct that represents a pet", + "properties": { + "name": { + "description": "name of the pet", + "type": "string" + }, + "tag": { + "description": "a tag of the pet, helps finding pet", + "type": "string" + }, + "id": { + "description": "unique indentifier", + "type": "integer" + } + } + } + } + } + } \ No newline at end of file diff --git a/examples/hero/generator/blank_generation/.gitignore b/examples/hero/generator/blank_generation/.gitignore new file mode 100644 index 000000000..29e2ed589 --- /dev/null +++ b/examples/hero/generator/blank_generation/.gitignore @@ -0,0 +1,2 @@ +.example_1_actor +.example_2_actor \ No newline at end of file diff --git a/examples/hero/generator/blank_generation/README.md b/examples/hero/generator/blank_generation/README.md new file mode 100644 index 000000000..e49b06c0e --- /dev/null +++ b/examples/hero/generator/blank_generation/README.md @@ -0,0 +1,19 @@ +## Blank Actor Generation Example + +This example shows how to generate a blank actor (unspecified, except for name). The generated actor module contains all the boilerplate code of an actor that can be compiled but lacks ant state or methods. + +Simply run: +``` +chmod +x *.vsh +example_1.vsh +example_2.vsh +``` + +### Examples + +There are two examples of blank actor generation. +- `example_1.vsh` generates the actor from a blank specification structure. +- `example_2.vsh` generates the actor from a blank OpenAPI Specification. + + +Read []() to learn how actor's are generated from specifications, and how the two example's differ. \ No newline at end of file diff --git a/examples/hero/generator/blank_generation/example_1.vsh b/examples/hero/generator/blank_generation/example_1.vsh new file mode 100644 index 000000000..73133089c --- /dev/null +++ b/examples/hero/generator/blank_generation/example_1.vsh @@ -0,0 +1,7 @@ +#!/usr/bin/env -S v -w -n -enable-globals run + +import freeflowuniverse.crystallib.hero.generation + +generation.generate_actor( + name: 'Example' +) \ No newline at end of file diff --git a/examples/hero/generator/blank_generation/example_2.vsh b/examples/hero/generator/blank_generation/example_2.vsh new file mode 100644 index 000000000..536e195bc --- /dev/null +++ b/examples/hero/generator/blank_generation/example_2.vsh @@ -0,0 +1,8 @@ +#!/usr/bin/env -S v -w -n -enable-globals run + +import freeflowuniverse.crystallib.hero.generation + +generation.generate_actor( + name: 'Example' + interfaces: [] +) \ No newline at end of file diff --git a/examples/hero/generator/openapi_generation/README.md b/examples/hero/generator/openapi_generation/README.md new file mode 100644 index 000000000..b2ae127bc --- /dev/null +++ b/examples/hero/generator/openapi_generation/README.md @@ -0,0 +1,26 @@ +# Hero Generation Example + +## Getting started + +Start by making all example scripts executable. +`chmod +x *.vsh` + +### Step 1: Generate specification + +### Step 2: Generate actor from specification + +The script below generates the actor's OpenAPI handler from a given OpenAPI Specification. The generated code is written to `handler.v` in the example actor's module. + +`generate_actor.vsh` + +### Step 3: Run actor + +The script below runs the actor's Redis RPC Queue Interface and uses the generated handler function to handle incoming RPCs. The Redis Interface listens to the RPC Queue assigned to the actor. + +`run_interface_procedure.vsh` + +### Step 3: Run server + +The script below runs the actor's RPC Queue Listener and uses the generated handler function to handle incoming RPCs. + +`run_interface_openapi.vsh` \ No newline at end of file diff --git a/examples/hero/generator/openapi_generation/example_actor/README.md b/examples/hero/generator/openapi_generation/example_actor/README.md new file mode 100644 index 000000000..13cc32a8a --- /dev/null +++ b/examples/hero/generator/openapi_generation/example_actor/README.md @@ -0,0 +1 @@ +# Example Actor \ No newline at end of file diff --git a/examples/hero/generator/openapi_generation/example_actor/actor.v b/examples/hero/generator/openapi_generation/example_actor/actor.v new file mode 100644 index 000000000..ad76d3879 --- /dev/null +++ b/examples/hero/generator/openapi_generation/example_actor/actor.v @@ -0,0 +1,36 @@ +module example_actor + +import os +import freeflowuniverse.crystallib.hero.baobab.actor {IActor, RunParams} +import freeflowuniverse.crystallib.web.openapi +import time + +const openapi_spec_path = '${os.dir(@FILE)}/specs/openapi.json' +const openapi_spec_json = os.read_file(openapi_spec_path) or { panic(err) } +const openapi_specification = openapi.json_decode(openapi_spec_json)! + +struct ExampleActor { + actor.Actor +} + +fn new() !ExampleActor { + return ExampleActor{ + actor.new('example') + } +} + +pub fn run() ! { + mut a_ := new()! + mut a := IActor(a_) + a.run()! +} + +pub fn run_server(params RunParams) ! { + mut a := new()! + mut server := actor.new_server( + redis_url: 'localhost:6379' + redis_queue: a.name + openapi_spec: openapi_specification + )! + server.run(params) +} \ No newline at end of file diff --git a/examples/hero/generator/openapi_generation/example_actor/actor_test.v b/examples/hero/generator/openapi_generation/example_actor/actor_test.v new file mode 100644 index 000000000..cab2e268f --- /dev/null +++ b/examples/hero/generator/openapi_generation/example_actor/actor_test.v @@ -0,0 +1,17 @@ +module example_actor + +const test_port = 8101 + +pub fn test_new() ! { + new() or { + return error('Failed to create actor:\n${err}') + } +} + +pub fn test_run() ! { + spawn run() +} + +pub fn test_run_server() ! { + spawn run_server(port: test_port) +} \ No newline at end of file diff --git a/examples/hero/generator/openapi_generation/example_actor/handle.v b/examples/hero/generator/openapi_generation/example_actor/handle.v new file mode 100644 index 000000000..dfc067ef4 --- /dev/null +++ b/examples/hero/generator/openapi_generation/example_actor/handle.v @@ -0,0 +1,5 @@ +module example_actor + +pub fn (mut a ExampleActor) handle(method string, data string) !string { + return data +} \ No newline at end of file diff --git a/examples/hero/generator/openapi_generation/example_actor/specs/openapi.json b/examples/hero/generator/openapi_generation/example_actor/specs/openapi.json new file mode 100644 index 000000000..77de98a39 --- /dev/null +++ b/examples/hero/generator/openapi_generation/example_actor/specs/openapi.json @@ -0,0 +1,346 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Pet Store API", + "description": "A sample API for a pet store", + "version": "1.0.0" + }, + "servers": [ + { + "url": "https://api.petstore.example.com/v1", + "description": "Production server" + }, + { + "url": "https://staging.petstore.example.com/v1", + "description": "Staging server" + } + ], + "paths": { + "/pets": { + "get": { + "summary": "List all pets", + "operationId": "listPets", + "parameters": [ + { + "name": "limit", + "in": "query", + "description": "Maximum number of pets to return", + "required": false, + "schema": { + "type": "integer", + "format": "int32" + } + } + ], + "responses": { + "200": { + "description": "A paginated list of pets", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Pets" + } + } + } + }, + "400": { + "description": "Invalid request" + } + } + }, + "post": { + "summary": "Create a new pet", + "operationId": "createPet", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewPet" + } + } + } + }, + "responses": { + "201": { + "description": "Pet created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Pet" + } + } + } + }, + "400": { + "description": "Invalid input" + } + } + } + }, + "/pets/{petId}": { + "get": { + "summary": "Get a pet by ID", + "operationId": "getPet", + "parameters": [ + { + "name": "petId", + "in": "path", + "description": "ID of the pet to retrieve", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "A pet", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Pet" + } + } + } + }, + "404": { + "description": "Pet not found" + } + } + }, + "delete": { + "summary": "Delete a pet by ID", + "operationId": "deletePet", + "parameters": [ + { + "name": "petId", + "in": "path", + "description": "ID of the pet to delete", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "204": { + "description": "Pet deleted" + }, + "404": { + "description": "Pet not found" + } + } + } + }, + "/orders": { + "get": { + "summary": "List all orders", + "operationId": "listOrders", + "responses": { + "200": { + "description": "A list of orders", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Order" + } + } + } + } + } + } + } + }, + "/orders/{orderId}": { + "get": { + "summary": "Get an order by ID", + "operationId": "getOrder", + "parameters": [ + { + "name": "orderId", + "in": "path", + "description": "ID of the order to retrieve", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "An order", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Order" + } + } + } + }, + "404": { + "description": "Order not found" + } + } + }, + "delete": { + "summary": "Delete an order by ID", + "operationId": "deleteOrder", + "parameters": [ + { + "name": "orderId", + "in": "path", + "description": "ID of the order to delete", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "204": { + "description": "Order deleted" + }, + "404": { + "description": "Order not found" + } + } + } + }, + "/users": { + "post": { + "summary": "Create a user", + "operationId": "createUser", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewUser" + } + } + } + }, + "responses": { + "201": { + "description": "User created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/User" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "Pet": { + "type": "object", + "required": ["id", "name"], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "name": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "NewPet": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "Pets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Pet" + } + }, + "Order": { + "type": "object", + "required": ["id", "petId", "quantity", "shipDate"], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "petId": { + "type": "integer", + "format": "int64" + }, + "quantity": { + "type": "integer", + "format": "int32" + }, + "shipDate": { + "type": "string", + "format": "date-time" + }, + "status": { + "type": "string", + "enum": ["placed", "approved", "delivered"] + }, + "complete": { + "type": "boolean" + } + } + }, + "User": { + "type": "object", + "required": ["id", "username"], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "username": { + "type": "string" + }, + "email": { + "type": "string" + }, + "phone": { + "type": "string" + } + } + }, + "NewUser": { + "type": "object", + "required": ["username"], + "properties": { + "username": { + "type": "string" + }, + "email": { + "type": "string" + }, + "phone": { + "type": "string" + } + } + } + } + } + } \ No newline at end of file diff --git a/examples/hero/generator/openapi_generation/generate_actor.vsh b/examples/hero/generator/openapi_generation/generate_actor.vsh new file mode 100644 index 000000000..f2aa5898a --- /dev/null +++ b/examples/hero/generator/openapi_generation/generate_actor.vsh @@ -0,0 +1 @@ +#!/usr/bin/env -S v -w -n -enable-globals run diff --git a/examples/hero/generator/openapi_generation/run_actor.vsh b/examples/hero/generator/openapi_generation/run_actor.vsh new file mode 100644 index 000000000..f2aa5898a --- /dev/null +++ b/examples/hero/generator/openapi_generation/run_actor.vsh @@ -0,0 +1 @@ +#!/usr/bin/env -S v -w -n -enable-globals run diff --git a/examples/hero/generator/openapi_generation/run_interface_procedure.vsh b/examples/hero/generator/openapi_generation/run_interface_procedure.vsh new file mode 100755 index 000000000..86380f42d --- /dev/null +++ b/examples/hero/generator/openapi_generation/run_interface_procedure.vsh @@ -0,0 +1,5 @@ +#!/usr/bin/env -S v -w -n -enable-globals run + +import example_actor + +example_actor.run_interface_procedure() \ No newline at end of file diff --git a/examples/hero/generator/openapi_generation/run_server.vsh b/examples/hero/generator/openapi_generation/run_server.vsh new file mode 100644 index 000000000..f2aa5898a --- /dev/null +++ b/examples/hero/generator/openapi_generation/run_server.vsh @@ -0,0 +1 @@ +#!/usr/bin/env -S v -w -n -enable-globals run diff --git a/examples/hero/openapi/README.md b/examples/hero/openapi/README.md new file mode 100644 index 000000000..86387d374 --- /dev/null +++ b/examples/hero/openapi/README.md @@ -0,0 +1,103 @@ +# OpenAPI Server with Redis-Based RPC and Actor + +This project demonstrates how to implement a system consisting of: + 1. An OpenAPI Server: Handles HTTP requests and translates them into procedure calls. + 2. A Redis-Based RPC Processor: Acts as the communication layer between the server and the actor. + 3. An Actor: Listens for RPC requests on a Redis queue and executes predefined procedures. + +## Features + • OpenAPI server to manage HTTP requests. + • Redis-based RPC mechanism for message passing. + • Actor pattern for executing and responding to RPC tasks. + +## Setup Instructions + +Prerequisites + • Redis installed and running on localhost:6379. + • V programming language installed. + +Steps to Run + +1. OpenAPI Specification + +Place the OpenAPI JSON specification file at: + +`data/openapi.json` + +This file defines the API endpoints and their parameters. + +2. Start the Redis Server + +Ensure Redis is running locally: + +redis-server + +3. Start the OpenAPI Server + +Run the OpenAPI server: + +`server.vsh` + +The server listens on port 8080 by default. + +4. Start the Actor + +Run the actor service: + +`actor.vsh` + +The actor listens to the procedure_queue for RPC messages. + +Usage + +API Endpoints + +The API supports operations like: + • Create a Pet: Adds a new pet. + • List Pets: Lists all pets or limits results. + • Get Pet by ID: Fetches a specific pet by ID. + • Delete Pet: Removes a pet by ID. + • Similar operations for users and orders. + +Use tools like curl, Postman, or a browser to interact with the endpoints. + +Example Requests + +Create a Pet + +curl -X POST http://localhost:8080/pets -d '{"name": "Buddy", "tag": "dog"}' -H "Content-Type: application/json" + +List Pets + +curl http://localhost:8080/pets + +## Code Overview + +1. OpenAPI Server + • Reads the OpenAPI JSON file. + • Maps HTTP requests to procedure calls using the operation ID. + • Sends procedure calls to the Redis RPC queue. + +2. Redis-Based RPC + • Implements a simple message queue using Redis. + • Encodes requests as JSON strings for transport. + +3. Actor + • Listens to the procedure_queue Redis queue. + • Executes tasks like managing pets, orders, and users. + • Responds with JSON-encoded results or errors. + +## Extending the System + +Add New Procedures + 1. Define new methods in the Actor to handle tasks. + 2. Add corresponding logic in the DataStore for storage operations. + 3. Update the OpenAPI JSON file to expose new endpoints. + +Modify Data Models + 1. Update the Pet, Order, and User structs as needed. + 2. Adjust the DataStore methods to handle the changes. + +Troubleshooting + • Redis Connection Issues: Ensure Redis is running and accessible on localhost:6379. + • JSON Parsing Errors: Validate the input JSON against the OpenAPI specification. diff --git a/examples/hero/openapi/actor b/examples/hero/openapi/actor new file mode 100755 index 000000000..1197e6ec8 Binary files /dev/null and b/examples/hero/openapi/actor differ diff --git a/examples/hero/openapi/actor.vsh b/examples/hero/openapi/actor.vsh new file mode 100755 index 000000000..776c5f2f7 --- /dev/null +++ b/examples/hero/openapi/actor.vsh @@ -0,0 +1,215 @@ +#!/usr/bin/env -S v -w -n -enable-globals run + +import os +import time +import veb +import json +import x.json2 +import net.http +import freeflowuniverse.crystallib.web.openapi {Server, Context, Request, Response} +import freeflowuniverse.crystallib.hero.processor {Processor, ProcedureCall, ProcedureResponse, ProcessParams} +import freeflowuniverse.crystallib.clients.redisclient + +@[heap] +struct Actor { +mut: + rpc redisclient.RedisRpc + data_store DataStore +} + +pub struct DataStore { +mut: + pets map[int]Pet + orders map[int]Order + users map[int]User +} + +struct Pet { + id int + name string + tag string +} + +struct Order { + id int + pet_id int + quantity int + ship_date string + status string + complete bool +} + +struct User { + id int + username string + email string + phone string +} + +// Entry point for the actor +fn main() { + mut redis := redisclient.new('localhost:6379') or {panic(err)} + mut rpc := redis.rpc_get('procedure_queue') + + mut actor := Actor{ + rpc: rpc + data_store: DataStore{} + } + + actor.listen() or {panic(err)} +} + +// Actor listens to the Redis queue for method invocations +fn (mut actor Actor) listen() ! { + println('Actor started and listening for tasks...') + for { + actor.rpc.process(actor.handle_method)! + time.sleep(time.millisecond * 100) // Prevent CPU spinning + } +} + +// Handle method invocations +fn (mut actor Actor) handle_method(cmd string, data string) !string { + println('debugzo received rpc ${cmd}:${data}') + param_anys := json2.raw_decode(data)!.arr() + match cmd { + 'listPets' { + pets := if param_anys.len == 0 { + actor.data_store.list_pets() + } else { + params := json.decode(ListPetParams, param_anys[0].str())! + actor.data_store.list_pets(params) + } + return json.encode(pets) + } + 'createPet' { + response := if param_anys.len == 0 { + return error('at least data expected') + } else if param_anys.len == 1 { + payload := json.decode(NewPet, param_anys[0].str())! + actor.data_store.create_pet(payload) + } else { + return error('expected 1 param, found too many') + } + // data := json.decode(NewPet, data) or { return error('Invalid pet data: $err') } + // created_pet := actor.data_store.create_pet(pet) + return json.encode(response) + } + 'getPet' { + response := if param_anys.len == 0 { + return error('at least data expected') + } else if param_anys.len == 1 { + payload := param_anys[0].int() + actor.data_store.get_pet(payload)! + } else { + return error('expected 1 param, found too many') + } + + return json.encode(response) + } + 'deletePet' { + params := json.decode(map[string]int, data) or { return error('Invalid params: $err') } + actor.data_store.delete_pet(params['petId']) or { return error('Pet not found: $err') } + return json.encode({'message': 'Pet deleted'}) + } + 'listOrders' { + orders := actor.data_store.list_orders() + return json.encode(orders) + } + 'getOrder' { + params := json.decode(map[string]int, data) or { return error('Invalid params: $err') } + order := actor.data_store.get_order(params['orderId']) or { + return error('Order not found: $err') + } + return json.encode(order) + } + 'deleteOrder' { + params := json.decode(map[string]int, data) or { return error('Invalid params: $err') } + actor.data_store.delete_order(params['orderId']) or { + return error('Order not found: $err') + } + return json.encode({'message': 'Order deleted'}) + } + 'createUser' { + user := json.decode(NewUser, data) or { return error('Invalid user data: $err') } + created_user := actor.data_store.create_user(user) + return json.encode(created_user) + } + else { + return error('Unknown method: $cmd') + } + } +} + +@[params] +pub struct ListPetParams { + limit u32 +} + +// DataStore methods for managing data +fn (mut store DataStore) list_pets(params ListPetParams) []Pet { + if params.limit > 0 { + if params.limit >= store.pets.values().len { + return store.pets.values() + } + return store.pets.values()[..params.limit] + } + return store.pets.values() +} + +fn (mut store DataStore) create_pet(new_pet NewPet) Pet { + id := store.pets.keys().len + 1 + pet := Pet{id: id, name: new_pet.name, tag: new_pet.tag} + store.pets[id] = pet + return pet +} + +fn (mut store DataStore) get_pet(id int) !Pet { + return store.pets[id] or { + return error('Pet with id ${id} not found.') + } +} + +fn (mut store DataStore) delete_pet(id int) ! { + if id in store.pets { + store.pets.delete(id) + return + } + return error('Pet not found') +} + +fn (mut store DataStore) list_orders() []Order { + return store.orders.values() +} + +fn (mut store DataStore) get_order(id int) !Order { + return store.orders[id] or { none } +} + +fn (mut store DataStore) delete_order(id int) ! { + if id in store.orders { + store.orders.delete(id) + return + } + return error('Order not found') +} + +fn (mut store DataStore) create_user(new_user NewUser) User { + id := store.users.keys().len + 1 + user := User{id: id, username: new_user.username, email: new_user.email, phone: new_user.phone} + store.users[id] = user + return user +} + +// NewPet struct for creating a pet +struct NewPet { + name string + tag string +} + +// NewUser struct for creating a user +struct NewUser { + username string + email string + phone string +} \ No newline at end of file diff --git a/examples/hero/openapi/data/openapi.json b/examples/hero/openapi/data/openapi.json new file mode 100644 index 000000000..c5ab2d9d4 --- /dev/null +++ b/examples/hero/openapi/data/openapi.json @@ -0,0 +1,346 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Pet Store API", + "description": "A sample API for a pet store", + "version": "1.0.0" + }, + "servers": [ + { + "url": "https://api.petstore.example.com/v1", + "description": "Production server" + }, + { + "url": "https://staging.petstore.example.com/v1", + "description": "Staging server" + } + ], + "paths": { + "/pets": { + "get": { + "summary": "List all pets", + "operationId": "listPets", + "parameters": [ + { + "name": "limit", + "in": "query", + "description": "Maximum number of pets to return", + "required": false, + "schema": { + "type": "integer", + "format": "int32" + } + } + ], + "responses": { + "200": { + "description": "A paginated list of pets", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Pets" + } + } + } + }, + "400": { + "description": "Invalid request" + } + } + }, + "post": { + "summary": "Create a new pet", + "operationId": "createPet", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewPet" + } + } + } + }, + "responses": { + "201": { + "description": "Pet created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Pet" + } + } + } + }, + "400": { + "description": "Invalid input" + } + } + } + }, + "/pets/{petId}": { + "get": { + "summary": "Get a pet by ID", + "operationId": "getPet", + "parameters": [ + { + "name": "petId", + "in": "path", + "description": "ID of the pet to retrieve", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "A pet", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Pet" + } + } + } + }, + "404": { + "description": "Pet not found" + } + } + }, + "delete": { + "summary": "Delete a pet by ID", + "operationId": "deletePet", + "parameters": [ + { + "name": "petId", + "in": "path", + "description": "ID of the pet to delete", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "204": { + "description": "Pet deleted" + }, + "404": { + "description": "Pet not found" + } + } + } + }, + "/orders": { + "get": { + "summary": "List all orders", + "operationId": "listOrders", + "responses": { + "200": { + "description": "A list of orders", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Order" + } + } + } + } + } + } + } + }, + "/orders/{orderId}": { + "get": { + "summary": "Get an order by ID", + "operationId": "getOrder", + "parameters": [ + { + "name": "orderId", + "in": "path", + "description": "ID of the order to retrieve", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "An order", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Order" + } + } + } + }, + "404": { + "description": "Order not found" + } + } + }, + "delete": { + "summary": "Delete an order by ID", + "operationId": "deleteOrder", + "parameters": [ + { + "name": "orderId", + "in": "path", + "description": "ID of the order to delete", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "204": { + "description": "Order deleted" + }, + "404": { + "description": "Order not found" + } + } + } + }, + "/users": { + "post": { + "summary": "Create a user", + "operationId": "createUser", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewUser" + } + } + } + }, + "responses": { + "201": { + "description": "User created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/User" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "Pet": { + "type": "object", + "required": ["id", "name"], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "name": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "NewPet": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "Pets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Pet" + } + }, + "Order": { + "type": "object", + "required": ["id", "petId", "quantity", "shipDate"], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "petId": { + "type": "integer", + "format": "int64" + }, + "quantity": { + "type": "integer", + "format": "int32" + }, + "shipDate": { + "type": "string", + "format": "date-time" + }, + "status": { + "type": "string", + "enum": ["placed", "approved", "delivered"] + }, + "complete": { + "type": "boolean" + } + } + }, + "User": { + "type": "object", + "required": ["id", "username"], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "username": { + "type": "string" + }, + "email": { + "type": "string" + }, + "phone": { + "type": "string" + } + } + }, + "NewUser": { + "type": "object", + "required": ["username"], + "properties": { + "username": { + "type": "string" + }, + "email": { + "type": "string" + }, + "phone": { + "type": "string" + } + } + } + } + } +} \ No newline at end of file diff --git a/examples/hero/openapi/server b/examples/hero/openapi/server new file mode 100755 index 000000000..86bac0a1b Binary files /dev/null and b/examples/hero/openapi/server differ diff --git a/examples/hero/openapi/server.vsh b/examples/hero/openapi/server.vsh new file mode 100755 index 000000000..572629253 --- /dev/null +++ b/examples/hero/openapi/server.vsh @@ -0,0 +1,141 @@ +#!/usr/bin/env -S v -w -n -enable-globals run + +import os +import time +import veb +import json +import x.json2 {Any} +import net.http +import freeflowuniverse.crystallib.data.jsonschema {Schema} +import freeflowuniverse.crystallib.web.openapi {Server, Context, Request, Response} +import freeflowuniverse.crystallib.hero.processor {Processor, ProcedureCall, ProcedureResponse, ProcessParams} +import freeflowuniverse.crystallib.clients.redisclient + +const spec_path = '${os.dir(@FILE)}/data/openapi.json' +const spec_json = os.read_file(spec_path) or { panic(err) } + +// Main function to start the server +fn main() { + // Initialize the Redis client and RPC mechanism + mut redis := redisclient.new('localhost:6379')! + mut rpc := redis.rpc_get('procedure_queue') + + // Initialize the server + mut server := &Server{ + specification: openapi.json_decode(spec_json)! + handler: Handler{ + processor: Processor{ + rpc: rpc + } + } + } + + // Start the server + veb.run[Server, Context](mut server, 8080) +} + +pub struct Handler { + mut: + processor Processor +} + +fn (mut handler Handler) handle(request Request) !Response { + // Convert incoming OpenAPI request to a procedure call + mut params := []string{} + + if request.arguments.len > 0 { + params = request.arguments.values().map(it.str()).clone() + } + + if request.body != '' { + params << request.body + } + + if request.parameters.len != 0 { + mut param_map := map[string]Any{} // Store parameters with correct types + + for param_name, param_value in request.parameters { + operation_param := request.operation.parameters.filter(it.name == param_name) + if operation_param.len > 0 { + param_schema := operation_param[0].schema as Schema + param_type := param_schema.typ + param_format := param_schema.format + + // Convert parameter value to corresponding type + match param_type { + 'integer' { + match param_format { + 'int32' { + param_map[param_name] = param_value.int() // Convert to int + } + 'int64' { + param_map[param_name] = param_value.i64() // Convert to i64 + } + else { + param_map[param_name] = param_value.int() // Default to int + } + } + } + 'string' { + param_map[param_name] = param_value // Already a string + } + 'boolean' { + param_map[param_name] = param_value.bool() // Convert to bool + } + 'number' { + match param_format { + 'float' { + param_map[param_name] = param_value.f32() // Convert to float + } + 'double' { + param_map[param_name] = param_value.f64() // Convert to double + } + else { + param_map[param_name] = param_value.f64() // Default to double + } + } + } + else { + param_map[param_name] = param_value // Leave as string for unknown types + } + } + } else { + // If the parameter is not defined in the OpenAPI operation, skip or log it + println('Unknown parameter: $param_name') + } + } + + // Encode the parameter map to JSON if needed + params << json.encode(param_map.str()) + } + + call := ProcedureCall{ + method: request.operation.operation_id + params: "[${params.join(',')}]" // Keep as a string since ProcedureCall expects a string + } + + // Process the procedure call + procedure_response := handler.processor.process( + call, + ProcessParams{ + timeout: 30 // Set timeout in seconds + } + ) or { + // Handle ProcedureError + if err is processor.ProcedureError { + return Response{ + status: http.status_from_int(err.code()) // Map ProcedureError reason to HTTP status code + body: json.encode({ + 'error': err.msg() + }) + } + } + return error('Unexpected error: $err') + } + + // Convert returned procedure response to OpenAPI response + return Response{ + status: http.Status.ok // Assuming success if no error + body: procedure_response.result + } +} \ No newline at end of file diff --git a/examples/walktest.vsh b/examples/walktest.vsh index 651a35de0..edb2a2fe0 100644 --- a/examples/walktest.vsh +++ b/examples/walktest.vsh @@ -1,28 +1,37 @@ +// This script walks through directories recursively looking for .collections files +// and extracts their collection names from the file content. import os fn main() { - for _, dirpath, _ in os.walk('.') { - for _, filename in os.listdir(dirpath) { - if filename.ends_with('.collections') { - println('Processing: ${filename}') - collection_name := get_collection_name(os.join_path(dirpath, filename)) - println('Collection name: ${collection_name}') - // ... do something with the collection_name - } - } - } + // Walk through all directories recursively starting from current directory ('.') + os.walk('.', fn (path string) { + // Check each file in the current directory + if path.ends_with('.collections') { + println('Processing: ${path}') + collection_name := get_collection_name(path) + println('Collection name: ${collection_name}') + // ... do something with the collection_name + } + }) } +// get_collection_name reads a file and extracts the collection name from it. +// If the file is empty or doesn't contain a name field, returns the base filename. fn get_collection_name(filepath string) string { - mut contents := os.read_file(filepath) or { return os.base(filepath) } - if contents.len == 0 { - return os.base(filepath) - } - lines := contents.split('\n') - for _, line in lines { - if line.trim().starts_with('name:') { - return line.trim()[5..].trim() // Extract text after "name:" - } - } - return os.base(filepath) + // Read the file contents, return base filename if reading fails + mut contents := os.read_file(filepath) or { return os.base(filepath) } + if contents.len == 0 { + return os.base(filepath) + } + + // Look for a line starting with 'name:' and extract the value + lines := contents.split('\n') + for line in lines { + if line.trim().starts_with('name:') { + return line.trim()[5..].trim() // Extract text after "name:" + } + } + + // Return base filename if no name field found + return os.base(filepath) } diff --git a/examples/web/openapi/README.md b/examples/web/openapi/README.md new file mode 100644 index 000000000..8129ce0d9 --- /dev/null +++ b/examples/web/openapi/README.md @@ -0,0 +1,9 @@ +### OpenAPI Example + +A simple example of how to use an OpenAPI server given an OpenAPI specification, with a custom handler to handle requests. The handler in this example echoes back the entire OpenAPI request object generated as the payload of the response. + +To run: +``` +chmod +x ${CODEROOT}/github/freeflowuniverse/crystallib/examples/web/openapi/server.vsh +${CODEROOT}/github/freeflowuniverse/crystallib/examples/web/openapi/server.vsh +``` \ No newline at end of file diff --git a/examples/web/openapi/data/openapi.json b/examples/web/openapi/data/openapi.json new file mode 100644 index 000000000..c5ab2d9d4 --- /dev/null +++ b/examples/web/openapi/data/openapi.json @@ -0,0 +1,346 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Pet Store API", + "description": "A sample API for a pet store", + "version": "1.0.0" + }, + "servers": [ + { + "url": "https://api.petstore.example.com/v1", + "description": "Production server" + }, + { + "url": "https://staging.petstore.example.com/v1", + "description": "Staging server" + } + ], + "paths": { + "/pets": { + "get": { + "summary": "List all pets", + "operationId": "listPets", + "parameters": [ + { + "name": "limit", + "in": "query", + "description": "Maximum number of pets to return", + "required": false, + "schema": { + "type": "integer", + "format": "int32" + } + } + ], + "responses": { + "200": { + "description": "A paginated list of pets", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Pets" + } + } + } + }, + "400": { + "description": "Invalid request" + } + } + }, + "post": { + "summary": "Create a new pet", + "operationId": "createPet", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewPet" + } + } + } + }, + "responses": { + "201": { + "description": "Pet created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Pet" + } + } + } + }, + "400": { + "description": "Invalid input" + } + } + } + }, + "/pets/{petId}": { + "get": { + "summary": "Get a pet by ID", + "operationId": "getPet", + "parameters": [ + { + "name": "petId", + "in": "path", + "description": "ID of the pet to retrieve", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "A pet", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Pet" + } + } + } + }, + "404": { + "description": "Pet not found" + } + } + }, + "delete": { + "summary": "Delete a pet by ID", + "operationId": "deletePet", + "parameters": [ + { + "name": "petId", + "in": "path", + "description": "ID of the pet to delete", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "204": { + "description": "Pet deleted" + }, + "404": { + "description": "Pet not found" + } + } + } + }, + "/orders": { + "get": { + "summary": "List all orders", + "operationId": "listOrders", + "responses": { + "200": { + "description": "A list of orders", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Order" + } + } + } + } + } + } + } + }, + "/orders/{orderId}": { + "get": { + "summary": "Get an order by ID", + "operationId": "getOrder", + "parameters": [ + { + "name": "orderId", + "in": "path", + "description": "ID of the order to retrieve", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "200": { + "description": "An order", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Order" + } + } + } + }, + "404": { + "description": "Order not found" + } + } + }, + "delete": { + "summary": "Delete an order by ID", + "operationId": "deleteOrder", + "parameters": [ + { + "name": "orderId", + "in": "path", + "description": "ID of the order to delete", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + } + ], + "responses": { + "204": { + "description": "Order deleted" + }, + "404": { + "description": "Order not found" + } + } + } + }, + "/users": { + "post": { + "summary": "Create a user", + "operationId": "createUser", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewUser" + } + } + } + }, + "responses": { + "201": { + "description": "User created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/User" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "Pet": { + "type": "object", + "required": ["id", "name"], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "name": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "NewPet": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "Pets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Pet" + } + }, + "Order": { + "type": "object", + "required": ["id", "petId", "quantity", "shipDate"], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "petId": { + "type": "integer", + "format": "int64" + }, + "quantity": { + "type": "integer", + "format": "int32" + }, + "shipDate": { + "type": "string", + "format": "date-time" + }, + "status": { + "type": "string", + "enum": ["placed", "approved", "delivered"] + }, + "complete": { + "type": "boolean" + } + } + }, + "User": { + "type": "object", + "required": ["id", "username"], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "username": { + "type": "string" + }, + "email": { + "type": "string" + }, + "phone": { + "type": "string" + } + } + }, + "NewUser": { + "type": "object", + "required": ["username"], + "properties": { + "username": { + "type": "string" + }, + "email": { + "type": "string" + }, + "phone": { + "type": "string" + } + } + } + } + } +} \ No newline at end of file diff --git a/examples/web/openapi/handler.v b/examples/web/openapi/handler.v new file mode 100644 index 000000000..79c607968 --- /dev/null +++ b/examples/web/openapi/handler.v @@ -0,0 +1,40 @@ +module main + +// Example route handler function +fn example_route_handler(request Request) !Response { + return Response{ + status_code: 200 + body: 'Hello, this is the example route!' + headers: {'Content-Type': 'text/plain'} + } +} + +// Example usage +fn main() { + // Initialize the handler with routes + handler := Handler{ + routes: { + '/example': example_route_handler + } + } + + // Create a sample request + request := Request{ + path: '/example' + method: 'GET' + body: '' + headers: {} + } + + // Handle the request + response := handler.handle(request) or { + eprintln('Error handling request: $err') + return + } + + // Print the response + println('Response:') + println('Status: $response.status_code') + println('Body: $response.body') + println('Headers: $response.headers') +} \ No newline at end of file diff --git a/examples/web/openapi/server.vsh b/examples/web/openapi/server.vsh new file mode 100755 index 000000000..842ce32ca --- /dev/null +++ b/examples/web/openapi/server.vsh @@ -0,0 +1,32 @@ +#!/usr/bin/env -S v -w -n -enable-globals run + +import os +import veb +import json +import freeflowuniverse.crystallib.web.openapi {Server, Context, Handler, Request, Response} + +const spec_path = '${os.dir(@FILE)}/data/openapi.json' +const spec_json = os.read_file(spec_path) or {panic(err)} + +// Main function to start the server +fn main() { + // Create the OpenAPI specification (mocked for now) + + // Initialize the server + mut server := &Server{ + specification: openapi.json_decode(spec_json)! + handler: ExampleHandler{} + } + + // Start the veb web server + veb.run[Server, Context](mut server, 8081) +} + +pub struct ExampleHandler{} + +fn (handler ExampleHandler) handle(request Request) !Response { + return Response { + status: .ok + body: '${request}' + } +} \ No newline at end of file diff --git a/examples/webdav/webdav.vsh b/examples/webdav/webdav.vsh new file mode 100755 index 000000000..b13bfb100 --- /dev/null +++ b/examples/webdav/webdav.vsh @@ -0,0 +1,29 @@ +#!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run + +import freeflowuniverse.crystallib.webdav +import freeflowuniverse.crystallib.core.pathlib +import time +import net.http +import encoding.base64 + +file_name := 'newfile.txt' +root_dir := '/tmp/webdav' + +username := "omda" +password := "password" +hashed_password := base64.encode_str('${username}:${password}') + +mut app := webdav.new_app(root_dir: root_dir, username: username, password: password) or { + eprintln('failed to create new server: ${err}') + exit(1) +} + +app.run(spawn_: true) + +time.sleep(1 * time.second) +mut p := pathlib.get_file(path: '${root_dir}/${file_name}', create: true)! +p.write('my new file')! + +mut req := http.new_request(.get, 'http://localhost:${app.server_port}/${file_name}', '') +req.add_custom_header('Authorization', 'Basic ${hashed_password}')! +req.do()! diff --git a/push_code.sh b/push_code.sh index 28b7d277d..91f17d998 100755 --- a/push_code.sh +++ b/push_code.sh @@ -7,4 +7,4 @@ cd "$MYDIR" echo "Enter the commit message:" read commit_message -git add . -A ; git commit -m "$commit_message"; git pull ; git push \ No newline at end of file +git add . -A ; git commit -m "$commit_message"; git pull ; git push diff --git a/research/gdrive/gdrive.v b/research/gdrive/gdrive.v index fc1dd15ec..8babea130 100644 --- a/research/gdrive/gdrive.v +++ b/research/gdrive/gdrive.v @@ -2,7 +2,7 @@ module gdrive import freeflowuniverse.crystallib.lang.python // import json -import freeflowuniverse.crystallib.core.dbfs +import freeflowuniverse.crystallib.data.dbfs import freeflowuniverse.crystallib.ui.console import freeflowuniverse.crystallib.core.pathlib import os diff --git a/research/vfs_research/mywebdav.vsh b/research/vfs_research/mywebdav.vsh index 76538ef2d..5188bd325 100755 --- a/research/vfs_research/mywebdav.vsh +++ b/research/vfs_research/mywebdav.vsh @@ -1,8 +1,71 @@ #!/usr/bin/env -S v -gc none -no-retry-compilation -cc tcc -d use_openssl -enable-globals run - import freeflowuniverse.crystallib.vfs.webdav +import freeflowuniverse.crystallib.core.pathlib +import time +import net.http +import encoding.base64 import os +file_name := 'newfile.txt' +root_dir := '/tmp/webdav' + +username := "admin" +password := "1234" +base64_encoded_creds := base64.encode_str('${username}:${password}') + +mut server := webdav.new_app(root_dir: root_dir, user_db: {username: password}) or { + eprintln('failed to create new server: ${err}') + exit(1) +} + +app.run(background: true) +time.sleep(500 * time.millisecond) + + +// get file +mut p := pathlib.get_file(path: '${root_dir}/${file_name}', create: true)! +p.write('my new file')! +mut req := http.new_request(.get, 'http://localhost:${server.server_port}/${file_name}','') +req.add_custom_header('Authorization', 'Basic ${base64_encoded_creds}')! +mut response := req.do()! +assert response.body == 'my new file' + +// create/update file +data2 := 'newdata' +req = http.new_request(.put, 'http://localhost:${server.server_port}/${file_name}', data2) +req.add_custom_header('Authorization', 'Basic ${base64_encoded_creds}')! +response = req.do()! +assert p.read()! == data2 + +file_name2 := 'newfile2.txt' +// copy file +req = http.new_request(.copy, 'http://localhost:${server.server_port}/${file_name}', '') +req.add_custom_header('Authorization', 'Basic ${base64_encoded_creds}')! +req.add_custom_header('Destination', 'http://localhost:${server.server_port}/${file_name2}')! +response = req.do()! +mut p2 := pathlib.get_file(path: '${root_dir}/${file_name2}')! +assert p2.read()! == data2 + +// move file +file_name3 := 'newfile3.txt' +req = http.new_request(.move, 'http://localhost:${server.server_port}/${file_name2}', '') +req.add_custom_header('Authorization', 'Basic ${base64_encoded_creds}')! +req.add_custom_header('Destination', 'http://localhost:${server.server_port}/${file_name3}')! +response = req.do()! +p2 = pathlib.get_file(path: '${root_dir}/${file_name3}')! +assert p2.read()! == data2 + +// delete file +req = http.new_request(.delete, 'http://localhost:${server.server_port}/${file_name3}', '') +req.add_custom_header('Authorization', 'Basic ${base64_encoded_creds}')! +response = req.do()! +assert !p2.exists() + -webdav.start(path:"/tmp")! \ No newline at end of file +// create directory +dir_name := 'newdir' +req = http.new_request(.mkcol, 'http://localhost:${server.server_port}/${dir_name}', '') +req.add_custom_header('Authorization', 'Basic ${base64_encoded_creds}')! +response = req.do()! +assert os.exists('${root_dir}/${dir_name}') diff --git a/tools/workspace b/tools/workspace new file mode 100644 index 000000000..d2ff8e8ea --- /dev/null +++ b/tools/workspace @@ -0,0 +1,23 @@ +{ + "folders": [ + { + "path": "/root/code/github/freeflowuniverse/crystallib/crystallib" + }, + { + "path": "/root/code/github/freeflowuniverse/crystallib/aiprompts" + }, + { + "path": "/root/code/github/freeflowuniverse/crystallib/c/root/code/github/freeflowuniverse/crystallib/hero" + }, + { + "path": "/root/code/github/freeflowuniverse/crystallib/scripts" + }, + { + "path": "/root/code/github/freeflowuniverse/crystallib/examples" + }, + { + "path": "/root/code/github/freeflowuniverse/crystallib/cli" + } + ], + "settings": {} +} \ No newline at end of file diff --git a/web3gw b/web3gw new file mode 160000 index 000000000..f8889dffc --- /dev/null +++ b/web3gw @@ -0,0 +1 @@ +Subproject commit f8889dffc17e62192d1ef66a0f7f89d355094d44