From 67dc583ff4b0a56626d6cd876052410fb434caf1 Mon Sep 17 00:00:00 2001 From: bag Date: Fri, 10 Jun 2016 08:21:46 +0300 Subject: [PATCH 01/23] readme --- readme.md | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/readme.md b/readme.md index eaeb26a..1b85b0a 100644 --- a/readme.md +++ b/readme.md @@ -43,8 +43,16 @@ cd FinXLog composer update ``` +## Optional: use with AMQP Queue for exchange high traffic +important: direct import is more quickly, if server has free resource - +### Install beanstool +```bash +wget https://github.com/src-d/beanstool/releases/download/v0.2.0/beanstool_v0.2.0_linux_amd64.tar.gz +tar -xvzf beanstool_v0.2.0_linux_amd64.tar.gz +sudo cp beanstool_v0.2.0_linux_amd64/beanstool /usr/local/bin/ +``` +------------------------------- # Using Daemon for import quotation: @@ -52,8 +60,6 @@ Daemon for import quotation: command/daemon/quotation_exchange2db.php ``` -## Optional: use with AMQP Queue for exchange high traffic -important: direct import is more quickly, if server has free resource ```bash command/daemon/quotation_exchange2amqp.php @@ -63,14 +69,7 @@ command/daemon/quotation_amqp2db.php ## Optional replacement for quotation_load.php is direct linux-way socket to amqp pipe for high performance source: https://github.com/src-d/beanstool -### Install beanstool -```bash -wget https://github.com/src-d/beanstool/releases/download/v0.2.0/beanstool_v0.2.0_linux_amd64.tar.gz -tar -xvzf beanstool_v0.2.0_linux_amd64.tar.gz -sudo cp beanstool_v0.2.0_linux_amd64/beanstool /usr/local/bin/ -``` -### Run -@todo: make auto-restart on network error + ```bash command/daemon/quotation_exchange2amqp.sh ``` From 25bb29b1cb2d8420ecfdedabd02d497c033cfa8d Mon Sep 17 00:00:00 2001 From: bag Date: Fri, 10 Jun 2016 08:23:45 +0300 Subject: [PATCH 02/23] config requirements --- config/app.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/app.php b/config/app.php index cc6ab5e..80b7628 100644 --- a/config/app.php +++ b/config/app.php @@ -6,3 +6,6 @@ $dot_env->required('FINXLOG_QUOTATION_SERVER_ADDRESS')->notEmpty(); $dot_env->required('FINXLOG_QUOTATION_SERVER_PORT')->notEmpty(); +$dot_env->required('FINXLOG_ELASTICO_PARAM')->notEmpty(); + +assert(json_decode(getenv('FINXLOG_ELASTICO_PARAM'))); \ No newline at end of file From 619295eb033e988ec1430641c57d6a56484d4c63 Mon Sep 17 00:00:00 2001 From: bag Date: Fri, 10 Jun 2016 13:36:00 +0300 Subject: [PATCH 03/23] config, readme --- .env.example | 4 ++-- readme.md | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index 7c071c0..c393acc 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,6 @@ PROJECT_NAME=FonXLog - -FINXLOG_QUOTATION_SERVER_ADDRESS=198.211.118.180 +FINXLOG_QUOTATION_SERVER_ADDRESS=178.62.145.164 +#FINXLOG_QUOTATION_SERVER_ADDRESS=198.211.118.180 FINXLOG_QUOTATION_SERVER_PORT=10000 FINXLOG_AMQP_SERVER_ADDRESS=127.0.0.1 diff --git a/readme.md b/readme.md index 1b85b0a..5f6f34b 100644 --- a/readme.md +++ b/readme.md @@ -1,15 +1,15 @@ -сделано: +Complete: - import - filter - save -инструменты: - - база данных ElasticSearch для быстрого поиска и bigdata -опционально: - - менеджер очередей BeanstalkD(AMQP) (по умолчанию работает без него). причина: для быстрой доставки клиентам и выдерживания "любой" нагрузки без анализа дублей(bash-скрипт) - - composer - - monolog - - .env окружение +Instruments: + - ElasticSearch for quick bigdata search +Oprional: + - BeanstalkD(AMQP, Queue manager). reason for use: quick delivery for "any" load with single stream(bash-scrpit) + - Composer, PSR-4 + - Monolog + - ".ENV" environment что надо сделать From 92501dc020fc5dfcf3e8ed6a6b511f3f6822d4b3 Mon Sep 17 00:00:00 2001 From: bag Date: Fri, 10 Jun 2016 17:25:26 +0300 Subject: [PATCH 04/23] readme, config, elasticsearch moholog, exception, fail queue --- .env.example | 5 +++ command/daemon/quotation_amqp2db.php | 19 ++++++--- command/daemon/quotation_exchange2amqp.sh | 2 +- command/stats_amqp.sh | 2 + config/app.php | 6 +++ config/dbg.php | 39 +++++++++++++++++++ config/pro.php | 27 +++++++++++++ readme.md | 25 +++++++----- src/Classes/Exception/ConnectionError.php | 2 +- src/Classes/Exception/Error.php | 2 +- src/Classes/Exception/WrongImport.php | 9 +++++ src/Classes/Exception/WrongParam.php | 9 ----- src/Classes/Exception/WrongParams.php | 27 +++++++++++++ src/Classes/Iface/ExceptionWithParams.php | 22 +++++++++++ ...xLogException.php => FinXLogException.php} | 0 src/Classes/Iface/ModuleImport.php | 4 ++ src/Classes/Iface/ModuleImportQueue.php | 31 +++++++++++++++ src/Classes/Module/Import/AbsQuotation.php | 26 ++++++++----- src/Classes/Module/Import/LoadQuotation.php | 24 ++++++++---- src/Classes/Module/Import/SaveQuotation.php | 19 ++++++--- .../Module/Import/Source/AbsSource.php | 12 ++++-- src/Classes/Module/Import/Source/Telnet.php | 38 +++++++++++------- src/Classes/Module/Logger.php | 38 ++++++++++++++++-- 23 files changed, 315 insertions(+), 73 deletions(-) create mode 100755 command/stats_amqp.sh create mode 100644 config/dbg.php create mode 100644 config/pro.php create mode 100644 src/Classes/Exception/WrongImport.php delete mode 100644 src/Classes/Exception/WrongParam.php create mode 100644 src/Classes/Exception/WrongParams.php create mode 100644 src/Classes/Iface/ExceptionWithParams.php rename src/Classes/Iface/{FinExLogException.php => FinXLogException.php} (100%) create mode 100644 src/Classes/Iface/ModuleImportQueue.php diff --git a/.env.example b/.env.example index c393acc..3b2c97c 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,6 @@ PROJECT_NAME=FonXLog +FINXLOG_DEBUG=0 + FINXLOG_QUOTATION_SERVER_ADDRESS=178.62.145.164 #FINXLOG_QUOTATION_SERVER_ADDRESS=198.211.118.180 FINXLOG_QUOTATION_SERVER_PORT=10000 @@ -9,3 +11,6 @@ FINXLOG_AMQP_TUBE_QUOTATION=finxlog_quotation FINXLOG_AMQP_TUBE_QUOTATION_FAIL=finxlog_quotation_fail FINXLOG_ELASTICO_PARAM={"host":"localhost"} +FINXLOG_FILTER_OTHER=1 + +#FINXLOG_ELASTICO_LOG_PARAM={"host":"localhost"} \ No newline at end of file diff --git a/command/daemon/quotation_amqp2db.php b/command/daemon/quotation_amqp2db.php index 6cf0165..9a3a2b4 100755 --- a/command/daemon/quotation_amqp2db.php +++ b/command/daemon/quotation_amqp2db.php @@ -6,10 +6,17 @@ */ require_once __DIR__ . '/../../vendor/autoload.php'; -$import = new FinXLog\Module\Import\SaveQuotation(); +$import = (new FinXLog\Module\Import\SaveQuotation()); -$import->run( - !empty($argv[1]) - ? $argv[1] - : null -); + +if (in_array('fail', $argv)) { + $import->setQueueConnector($import->getFailQueueConnector()); +} +$count = null; +foreach ($argv as $a) { + if (is_numeric($a) && $a > 0) { + $count = $a; + } +} + +$import->run($count); diff --git a/command/daemon/quotation_exchange2amqp.sh b/command/daemon/quotation_exchange2amqp.sh index c65b236..b554fbf 100755 --- a/command/daemon/quotation_exchange2amqp.sh +++ b/command/daemon/quotation_exchange2amqp.sh @@ -6,4 +6,4 @@ export $(cat $FINXLOG_ROOT_DIR.env | grep -P '^FINXLOG_(QUOTATION|AMQP)') #export | grep FIN # telnet > amqp -telnet $FINXLOG_QUOTATION_SERVER_ADDRESS $FINXLOG_QUOTATION_SERVER_PORT | xargs -I {} echo beanstool put -t=$FINXLOG_AMQP_TUBE -b {} +telnet $FINXLOG_QUOTATION_SERVER_ADDRESS $FINXLOG_QUOTATION_SERVER_PORT | grep '=' | xargs -I {} beanstool put -t=$FINXLOG_AMQP_TUBE_QUOTATION -b {} diff --git a/command/stats_amqp.sh b/command/stats_amqp.sh new file mode 100755 index 0000000..7133b3f --- /dev/null +++ b/command/stats_amqp.sh @@ -0,0 +1,2 @@ +#!/bin/bash +beanstool stats \ No newline at end of file diff --git a/config/app.php b/config/app.php index 80b7628..4bcaeda 100644 --- a/config/app.php +++ b/config/app.php @@ -8,4 +8,10 @@ $dot_env->required('FINXLOG_QUOTATION_SERVER_PORT')->notEmpty(); $dot_env->required('FINXLOG_ELASTICO_PARAM')->notEmpty(); +if (getenv('FINXLOG_DEBUG')) { + require __DIR__ . '/dbg.php'; +} else { + require __DIR__ . '/pro.php'; +} + assert(json_decode(getenv('FINXLOG_ELASTICO_PARAM'))); \ No newline at end of file diff --git a/config/dbg.php b/config/dbg.php new file mode 100644 index 0000000..d1db1cc --- /dev/null +++ b/config/dbg.php @@ -0,0 +1,39 @@ +warning($string . ";code:$code at $file:$line"); + + return true; + } +); + +/* +assert(false,'assert'); +trigger_error('trigger'); +echo "\nend\n"; +*/ \ No newline at end of file diff --git a/config/pro.php b/config/pro.php new file mode 100644 index 0000000..76480bf --- /dev/null +++ b/config/pro.php @@ -0,0 +1,27 @@ +getStatus()->getIndexNames())); + + /** + * log message in elasticsearch + */ + \FinXLog\Module\Logger::log() + ->pushHandler( + new \Monolog\Handler\ElasticSearchHandler( + $logClient, + [], + Monolog\Logger::INFO + ) + ); +} catch (\Exception $e) { } \ No newline at end of file diff --git a/readme.md b/readme.md index 5f6f34b..e3c3be4 100644 --- a/readme.md +++ b/readme.md @@ -43,15 +43,6 @@ cd FinXLog composer update ``` -## Optional: use with AMQP Queue for exchange high traffic -important: direct import is more quickly, if server has free resource - -### Install beanstool -```bash -wget https://github.com/src-d/beanstool/releases/download/v0.2.0/beanstool_v0.2.0_linux_amd64.tar.gz -tar -xvzf beanstool_v0.2.0_linux_amd64.tar.gz -sudo cp beanstool_v0.2.0_linux_amd64/beanstool /usr/local/bin/ -``` ------------------------------- # Using Daemon for import quotation: @@ -60,13 +51,19 @@ Daemon for import quotation: command/daemon/quotation_exchange2db.php ``` +## Optional: use with AMQP Queue for exchange high traffic +important: direct import is more quickly, if server has free resource ```bash +#run each daemons command/daemon/quotation_exchange2amqp.php + +#load daemons can work on other servers with fork command/daemon/quotation_amqp2db.php +command/daemon/quotation_amqp2db.php fail ``` -## Optional replacement for quotation_load.php +## Optional BASH replacement for quotation_load.php is direct linux-way socket to amqp pipe for high performance source: https://github.com/src-d/beanstool @@ -74,4 +71,12 @@ source: https://github.com/src-d/beanstool command/daemon/quotation_exchange2amqp.sh ``` +### Install beanstool +```bash +wget https://github.com/src-d/beanstool/releases/download/v0.2.0/beanstool_v0.2.0_linux_amd64.tar.gz +tar -xvzf beanstool_v0.2.0_linux_amd64.tar.gz +sudo cp beanstool_v0.2.0_linux_amd64/beanstool /usr/local/bin/ +``` + @todo: make quotation_exchange2db.sh +@todo: lock for parallel import \ No newline at end of file diff --git a/src/Classes/Exception/ConnectionError.php b/src/Classes/Exception/ConnectionError.php index 6d89d4c..9bc24df 100644 --- a/src/Classes/Exception/ConnectionError.php +++ b/src/Classes/Exception/ConnectionError.php @@ -3,7 +3,7 @@ use FinXLog\Iface; -class ConnectionError extends \Exception implements Iface\FinXLogException +class ConnectionError extends Error { } \ No newline at end of file diff --git a/src/Classes/Exception/Error.php b/src/Classes/Exception/Error.php index e30b21c..d3b854f 100644 --- a/src/Classes/Exception/Error.php +++ b/src/Classes/Exception/Error.php @@ -3,7 +3,7 @@ use FinXLog\Iface; -class Error extends \Exception implements Iface\FinXLogException +class Error extends \ErrorException implements Iface\FinXLogException { } \ No newline at end of file diff --git a/src/Classes/Exception/WrongImport.php b/src/Classes/Exception/WrongImport.php new file mode 100644 index 0000000..d42dbd4 --- /dev/null +++ b/src/Classes/Exception/WrongImport.php @@ -0,0 +1,9 @@ +params; + } + + public function setParams(array $params) + { + $this->params = $params; + + return $this; + } + public function addParams(array $params) + { + $this->params = $params + $this->params; + + return $this; + } +} \ No newline at end of file diff --git a/src/Classes/Iface/ExceptionWithParams.php b/src/Classes/Iface/ExceptionWithParams.php new file mode 100644 index 0000000..bcd6b2f --- /dev/null +++ b/src/Classes/Iface/ExceptionWithParams.php @@ -0,0 +1,22 @@ +fail_queue_connector) { + if (!$this->fail_queue_connector) { $this->fail_queue_connector = new Connector\Queue( getenv('FINXLOG_AMQP_TUBE_QUOTATION_FAIL') ); @@ -51,13 +49,19 @@ public function getDefaultQueueConnector() public function failQueu(Job $queue_job) { - $this->getFailQueueConnector() - ->put( - $queue_job->getData() - ); + if ($this->getFailQueueConnector()) { + $this->getFailQueueConnector() + ->put( + $queue_job->getData() + ); + } - $this->getQueueConnector() - ->delete($queue_job); + if ($this->getQueueConnector()) { + $this->getQueueConnector() + ->delete($queue_job); + } + + return $this; } public function importQuotation($string) @@ -66,6 +70,8 @@ public function importQuotation($string) ->save( Import\Source\Telnet::getFromRaw($string) ); + + return $this; } } \ No newline at end of file diff --git a/src/Classes/Module/Import/LoadQuotation.php b/src/Classes/Module/Import/LoadQuotation.php index a996b84..d5ce693 100644 --- a/src/Classes/Module/Import/LoadQuotation.php +++ b/src/Classes/Module/Import/LoadQuotation.php @@ -2,12 +2,10 @@ namespace FinXLog\Module\Import; use FinXLog\Exception\ConnectionError; -use FinXLog\Exception\ErrorParam; use FinXLog\Iface; use FinXLog\Module\Connector; use FinXLog\Module\Logger; use FinXLog\Traits; -use FinXLog\Helper; use FinXLog\Model; /** @@ -33,11 +31,23 @@ public function getDefaultConnector() public function run($limit = null) { while ($limit === null || --$limit >= 0) { - $this->saveJob( - $this->getConnector() - ->getQuotation() - ); - Logger::log()->info('job done'); + $import = null; + try { + $import = $this->getConnector() + ->getQuotation(); + } catch (\Exception $e) { + Logger::log()->debug('-'); + Logger::error('LoadQuotation getQuotation error', $e); + } + if ($import) { + try { + $this->saveJob($import); + Logger::log()->debug('+'); + } catch (\Exception $e) { + Logger::log()->debug('-'); + Logger::error('LoadQuotation saveJob. lost: ' . var_export($import, true), $e); + } + } } return $this; diff --git a/src/Classes/Module/Import/SaveQuotation.php b/src/Classes/Module/Import/SaveQuotation.php index 8d7da80..a7659c3 100644 --- a/src/Classes/Module/Import/SaveQuotation.php +++ b/src/Classes/Module/Import/SaveQuotation.php @@ -1,10 +1,11 @@ watch(); while ($limit === null || --$limit >= 0) { - $queu_job = $this->getQueueConnector()->reserve(); + $queue_job = $this->getQueueConnector()->reserve(); try { - $this->importQuotation($queu_job->getData()); + $this->importQuotation($queue_job->getData()); + Logger::log()->debug('+'); $this->getQueueConnector() - ->delete($queu_job); + ->delete($queue_job); + } catch (Exception\WrongImport $e) { + Logger::log()->debug('-'); + Logger::error('SaveQuotation WrongImport', $e); + $this->getQueueConnector() + ->delete($queue_job); } catch (\Exception $e) { - $this->failQueu($queu_job); + Logger::log()->debug('-'); + Logger::error('SaveQuotation exception', $e); + $this->failQueu($queue_job); } } diff --git a/src/Classes/Module/Import/Source/AbsSource.php b/src/Classes/Module/Import/Source/AbsSource.php index 37ff91a..f678940 100644 --- a/src/Classes/Module/Import/Source/AbsSource.php +++ b/src/Classes/Module/Import/Source/AbsSource.php @@ -1,6 +1,7 @@ setParams(['quotation' =>$result]); } if (!strtotime($result['T'])) { - throw new Exception\WrongParam('is not a valid datetime: ' . $result['T']); + throw (new Exception\WrongParams('is not a valid datetime: ' . $result['T'])) + ->setParams(['T' => $result['T']]); } } @@ -59,7 +62,8 @@ public static function getTime($string) { $timestamp = strtotime($string); if (!$timestamp) { - throw new Exception\WrongParam('is not a valid datetime: ' . $string); + throw (new Exception\WrongParams('is not a valid datetime: ' . $string)) + ->setParams(['T' => $string]); } return date(static::DATETIME_FORMAT, $timestamp); } diff --git a/src/Classes/Module/Import/Source/Telnet.php b/src/Classes/Module/Import/Source/Telnet.php index ff122c8..ab993b2 100644 --- a/src/Classes/Module/Import/Source/Telnet.php +++ b/src/Classes/Module/Import/Source/Telnet.php @@ -2,32 +2,40 @@ namespace FinXLog\Module\Import\Source; use FinXLog\Exception; +use FinXLog\Iface; class Telnet extends AbsSource { /** * @param $string * @return array - * @throws Exception\WrongParam + * @throws Exception\WrongParams */ public static function getFromRaw($string) { - assert(is_string($string)); + try { + assert(is_string($string)); - $field = explode( - ';', - trim($string) - ); - assert(count($field) == 3); - $result = []; - foreach ($field as $cur) { - list($name, $value) = explode('=', $cur); - $result[$name] = $value; - } - static::validate($result); - $result = static::prepare($result); - $result = static::filter($result); + $field = explode( + ';', + trim($string) + ); + if (count($field) < 3) { + throw new Exception\WrongImport('wrong import'); + } + assert(count($field) == 3); + $result = []; + foreach ($field as $cur) { + list($name, $value) = explode('=', $cur); + $result[$name] = $value; + } + static::validate($result); + $result = static::prepare($result); + $result = static::filter($result); + } catch (Iface\ExceptionWithParams $e) { + throw $e->addParams(['import' => $string]); + } return $result; } } \ No newline at end of file diff --git a/src/Classes/Module/Logger.php b/src/Classes/Module/Logger.php index 905ceee..924b0aa 100644 --- a/src/Classes/Module/Logger.php +++ b/src/Classes/Module/Logger.php @@ -1,5 +1,6 @@ getLogger(); } + public static function error($message, $context) + { + if ($context instanceof \Exception) { + $context = [ + 'exception' => get_class($context), + 'trace' =>$context->getTraceAsString() + ]; + if ($context instanceof Iface\ExceptionWithParams) { + $context = [ + 'params' => $context->getParams() + ]; + } + } + + return static::log()->err($message, $context); + } + public function setLogger(\Psr\Log\LoggerInterface $logger) { @@ -37,6 +58,11 @@ public function setLogger(\Psr\Log\LoggerInterface $logger) return $this; } + private function getDefaultCliFormater() + { + return new Monolog\Formatter\LineFormatter("%message%"); + } + private function getLoggerDefault() { $logger = new Monolog\Logger('Language'); @@ -48,11 +74,15 @@ private function getLoggerDefault() ) ) ->pushHandler( - new Monolog\Handler\StreamHandler( - 'php://stdout', - Monolog\Logger::INFO, - false + ( + new Monolog\Handler\StreamHandler( + 'php://stdout', + getenv('FINXLOG_DEBUG') ? Monolog\Logger::DEBUG : Monolog\Logger::INFO + ) ) + ->setFormatter( + $this->getDefaultCliFormater() + ) ); return $logger; From 9b8b4f81f54d2d36eed3ebe1d7a1c4d11125ffd4 Mon Sep 17 00:00:00 2001 From: bag Date: Fri, 10 Jun 2016 22:56:25 +0300 Subject: [PATCH 05/23] readme --- command/{stats_amqp.sh => stats/amqp.sh} | 0 readme.md | 29 ++++++++++++------------ 2 files changed, 14 insertions(+), 15 deletions(-) rename command/{stats_amqp.sh => stats/amqp.sh} (100%) diff --git a/command/stats_amqp.sh b/command/stats/amqp.sh similarity index 100% rename from command/stats_amqp.sh rename to command/stats/amqp.sh diff --git a/readme.md b/readme.md index e3c3be4..b83f596 100644 --- a/readme.md +++ b/readme.md @@ -1,31 +1,33 @@ +v 0.9.1 Complete: - import - filter - save + - queue + - elastic log Instruments: - - ElasticSearch for quick bigdata search -Oprional: - - BeanstalkD(AMQP, Queue manager). reason for use: quick delivery for "any" load with single stream(bash-scrpit) + - ElasticSearch for quick big data search - Composer, PSR-4 - Monolog - ".ENV" environment +Optional: + - BeanstalkD(AMQP Queue manager). reason for use: quick delivery for "any" load with single stream(bash-scrpit) + + что надо сделать - модуль для аггрегации данных https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html - браузерный static интерфейс с highcharts, etc - простое json API + - испрользовать те же очереди для доставки клиентам(websocket, socket.io) параллельно с сохранением в БД -опционально и причина использования менеджера очередей: -параллельно с сохранением данных в БД, отправлять данные клиентам данные в реальном времени(поступления) через websocket или socket.io. -настройка преоретизации - сначала в бд, потом к клиентам - -можно: -опционально сохранение в SQL если "появятся" SQL задачи -использование ElasticSearch для всех логов - - +Optional: + - sql db + - make quotation_exchange2db.sh + - lock for parallel import + # Install ```bash #install requirements @@ -77,6 +79,3 @@ wget https://github.com/src-d/beanstool/releases/download/v0.2.0/beanstool_v0.2. tar -xvzf beanstool_v0.2.0_linux_amd64.tar.gz sudo cp beanstool_v0.2.0_linux_amd64/beanstool /usr/local/bin/ ``` - -@todo: make quotation_exchange2db.sh -@todo: lock for parallel import \ No newline at end of file From f2d99c4ce381b7bfd33aef3061ecb67d93d3811b Mon Sep 17 00:00:00 2001 From: bag Date: Sun, 12 Jun 2016 07:40:20 +0300 Subject: [PATCH 06/23] ElasticSearch agg, doji, naming: elastico => elastica, telnet => netcat --- .env.example | 4 +- command/daemon/quotation_amqp2db.php | 3 +- command/daemon/quotation_exchange2amqp.php | 2 +- command/daemon/quotation_exchange2amqp.sh | 7 +- command/daemon/quotation_exchange2db.php | 2 +- config/app.php | 4 +- config/pro.php | 6 +- ...icoConnector.php => ElasticaConnector.php} | 2 +- ...ElasticoModel.php => AbsElasticaModel.php} | 29 ++++- src/Classes/Model/Quotation.php | 2 +- src/Classes/Model/QuotationAgg.php | 123 ++++++++++++++++++ .../Connector/{Elastico.php => Elastica.php} | 8 +- .../AbsQuotation.php | 6 +- .../LoadQuotation.php | 5 +- .../SaveQuotation.php | 4 +- .../Source/AbsSource.php | 9 +- .../Source/Telnet.php | 2 +- src/Classes/Module/Logger.php | 2 +- tests/ConnectorQuotationTest.php | 5 +- tests/ModelQuotationAggTest.php | 42 ++++++ tests/ModelQuotationTest.php | 28 ++++ 21 files changed, 257 insertions(+), 38 deletions(-) rename src/Classes/Iface/{ElasticoConnector.php => ElasticaConnector.php} (90%) rename src/Classes/Model/{AbsElasticoModel.php => AbsElasticaModel.php} (73%) create mode 100644 src/Classes/Model/QuotationAgg.php rename src/Classes/Module/Connector/{Elastico.php => Elastica.php} (67%) rename src/Classes/Module/{Import => ImportQuotation}/AbsQuotation.php (91%) rename src/Classes/Module/{Import => ImportQuotation}/LoadQuotation.php (95%) rename src/Classes/Module/{Import => ImportQuotation}/SaveQuotation.php (93%) rename src/Classes/Module/{Import => ImportQuotation}/Source/AbsSource.php (90%) rename src/Classes/Module/{Import => ImportQuotation}/Source/Telnet.php (95%) create mode 100644 tests/ModelQuotationAggTest.php create mode 100644 tests/ModelQuotationTest.php diff --git a/.env.example b/.env.example index 3b2c97c..36c4afb 100644 --- a/.env.example +++ b/.env.example @@ -10,7 +10,7 @@ FINXLOG_AMQP_SERVER_ADDRESS=127.0.0.1 FINXLOG_AMQP_TUBE_QUOTATION=finxlog_quotation FINXLOG_AMQP_TUBE_QUOTATION_FAIL=finxlog_quotation_fail -FINXLOG_ELASTICO_PARAM={"host":"localhost"} +FINXLOG_ELASTICA_PARAM={"host":"localhost"} FINXLOG_FILTER_OTHER=1 -#FINXLOG_ELASTICO_LOG_PARAM={"host":"localhost"} \ No newline at end of file +#FINXLOG_ELASTICA_LOG_PARAM={"host":"localhost"} \ No newline at end of file diff --git a/command/daemon/quotation_amqp2db.php b/command/daemon/quotation_amqp2db.php index 9a3a2b4..28a84c0 100755 --- a/command/daemon/quotation_amqp2db.php +++ b/command/daemon/quotation_amqp2db.php @@ -6,8 +6,7 @@ */ require_once __DIR__ . '/../../vendor/autoload.php'; -$import = (new FinXLog\Module\Import\SaveQuotation()); - +$import = (new FinXLog\Module\ImportQuotation\SaveQuotation()); if (in_array('fail', $argv)) { $import->setQueueConnector($import->getFailQueueConnector()); diff --git a/command/daemon/quotation_exchange2amqp.php b/command/daemon/quotation_exchange2amqp.php index bec1d34..7a6b30c 100755 --- a/command/daemon/quotation_exchange2amqp.php +++ b/command/daemon/quotation_exchange2amqp.php @@ -6,7 +6,7 @@ */ require_once __DIR__ . '/../../vendor/autoload.php'; -$import = new FinXLog\Module\Import\LoadQuotation(); +$import = new FinXLog\Module\ImportQuotation\LoadQuotation(); $import->run( !empty($argv[1]) diff --git a/command/daemon/quotation_exchange2amqp.sh b/command/daemon/quotation_exchange2amqp.sh index b554fbf..01f5b00 100755 --- a/command/daemon/quotation_exchange2amqp.sh +++ b/command/daemon/quotation_exchange2amqp.sh @@ -5,5 +5,8 @@ export $(cat $FINXLOG_ROOT_DIR.env | grep -P '^FINXLOG_(QUOTATION|AMQP)') #export | grep FIN -# telnet > amqp -telnet $FINXLOG_QUOTATION_SERVER_ADDRESS $FINXLOG_QUOTATION_SERVER_PORT | grep '=' | xargs -I {} beanstool put -t=$FINXLOG_AMQP_TUBE_QUOTATION -b {} +# netcat > amqp +while : +do + netcat $FINXLOG_QUOTATION_SERVER_ADDRESS $FINXLOG_QUOTATION_SERVER_PORT | xargs -I {} beanstool put -t=$FINXLOG_AMQP_TUBE_QUOTATION -b {} +done \ No newline at end of file diff --git a/command/daemon/quotation_exchange2db.php b/command/daemon/quotation_exchange2db.php index 90c9d96..d2f48cc 100755 --- a/command/daemon/quotation_exchange2db.php +++ b/command/daemon/quotation_exchange2db.php @@ -6,7 +6,7 @@ */ require_once __DIR__ . '/../../vendor/autoload.php'; -$import = new FinXLog\Module\Import\LoadQuotation(); +$import = new FinXLog\Module\ImportQuotation\LoadQuotation(); $import ->setWorkWithSwitch(false) diff --git a/config/app.php b/config/app.php index 4bcaeda..5e3e67e 100644 --- a/config/app.php +++ b/config/app.php @@ -6,7 +6,7 @@ $dot_env->required('FINXLOG_QUOTATION_SERVER_ADDRESS')->notEmpty(); $dot_env->required('FINXLOG_QUOTATION_SERVER_PORT')->notEmpty(); -$dot_env->required('FINXLOG_ELASTICO_PARAM')->notEmpty(); +$dot_env->required('FINXLOG_ELASTICA_PARAM')->notEmpty(); if (getenv('FINXLOG_DEBUG')) { require __DIR__ . '/dbg.php'; @@ -14,4 +14,4 @@ require __DIR__ . '/pro.php'; } -assert(json_decode(getenv('FINXLOG_ELASTICO_PARAM'))); \ No newline at end of file +assert(json_decode(getenv('FINXLOG_ELASTICA_PARAM'))); \ No newline at end of file diff --git a/config/pro.php b/config/pro.php index 76480bf..fcca7ec 100644 --- a/config/pro.php +++ b/config/pro.php @@ -4,9 +4,9 @@ assert(\FinXLog\Module\Logger::log() instanceof \Monolog\Logger); $logClient = new \Elastica\Client( json_decode( - getenv('FINXLOG_ELASTICO_LOG_PARAM') - ? getenv('FINXLOG_ELASTICO_LOG_PARAM') - : getenv('FINXLOG_ELASTICO_PARAM'), + getenv('FINXLOG_ELASTICA_LOG_PARAM') + ? getenv('FINXLOG_ELASTICA_LOG_PARAM') + : getenv('FINXLOG_ELASTICA_PARAM'), true ) ); diff --git a/src/Classes/Iface/ElasticoConnector.php b/src/Classes/Iface/ElasticaConnector.php similarity index 90% rename from src/Classes/Iface/ElasticoConnector.php rename to src/Classes/Iface/ElasticaConnector.php index 9f97314..f24269b 100644 --- a/src/Classes/Iface/ElasticoConnector.php +++ b/src/Classes/Iface/ElasticaConnector.php @@ -1,7 +1,7 @@ getConnector()->getConnector(); } + /** + * @return \Elastica\Index + */ + public function getElasticIndex() + { + return $this->getDb() + ->getIndex($this->index); + } + + //@todo for simple query + //public function getResult(Query $query); + + /** + * @return \Elastica\ResultSet + */ + public function getResponse(\Elastica\Query $query) + { + return $this->getElasticIndex() + ->search($query); + } + public function checkConnector(Iface\Connector $connector) { assert($connector->getConnector() instanceof \Elastica\Client); diff --git a/src/Classes/Model/Quotation.php b/src/Classes/Model/Quotation.php index 947ef09..8bbd4c0 100644 --- a/src/Classes/Model/Quotation.php +++ b/src/Classes/Model/Quotation.php @@ -2,7 +2,7 @@ namespace FinXLog\Model; use FinXLog\Traits; -class Quotation extends AbsElasticoModel +class Quotation extends AbsElasticaModel { protected $index = 'quotation'; protected $type = 'quotation'; diff --git a/src/Classes/Model/QuotationAgg.php b/src/Classes/Model/QuotationAgg.php new file mode 100644 index 0000000..7d5b7ff --- /dev/null +++ b/src/Classes/Model/QuotationAgg.php @@ -0,0 +1,123 @@ + stats + */ + private $query_doji = '{ + "query": { + "bool": { + "must": [ + { + "query_string": { + "default_field": "S", + "query": "BTCUSD" + } + } + ] + } + }, + "from": 0, + "size": 0, + "sort": [], + "aggs": { + "date": { + "date_histogram": { + "min_doc_count": 1, + "field": "T", + "interval": "1d", + "order": { + "_key": "desc" + } + }, + "aggs": { + "stat": { + "extended_stats": { + "field": "B", + "sigma": 3 + } + }, + "last": { + "terms": { + "size": 1, + "field": "T", + "order": { + "_term": "desc" + } + }, + "aggs": { + "avg": { + "avg": { + "field": "B" + } + } + } + }, + "first": { + "terms": { + "size": 1, + "field": "T", + "order": { + "_term": "asc" + } + }, + "aggs": { + "avg": { + "avg": { + "field": "B" + } + } + } + } + } + } + } + }'; + + /** + * get DOJI by exchange subject (japanese candlesticks) + * [date][min,max,first,last] + * @param string $subject exchange subject(EURUSD, USDBTC) + * @param string $interval (period) + * @return mixed + */ + public function getDoji($subject, $interval = 'day') + { + return $this->getResult( + $this->getDojiQuery( + $subject, + $interval + ) + ); + } + + /** + * return + * @param Query $query + * @return array + */ + public function getResult(Query $query) + { + return current( //1st agg name is not important + $this->getResponse($query) + ->getAggregations() + )['buckets']; + } + + public function getDojiQuery($subject, $interval = 'day') + { + $query = json_decode($this->query_doji, true); + + $query['query']['bool']['must'][0]['query_string']['query'] = $subject; + $query['aggs']['date']['date_histogram']['interval'] = $interval; + + return new Query($query); + } +} diff --git a/src/Classes/Module/Connector/Elastico.php b/src/Classes/Module/Connector/Elastica.php similarity index 67% rename from src/Classes/Module/Connector/Elastico.php rename to src/Classes/Module/Connector/Elastica.php index 95b051f..fc7643f 100644 --- a/src/Classes/Module/Connector/Elastico.php +++ b/src/Classes/Module/Connector/Elastica.php @@ -8,7 +8,7 @@ * Class Db * @package FinXLog\Module\Connector */ -class Elastico implements Iface\Connector +class Elastica implements Iface\Connector { use Traits\WithConnectorRaw; protected $param = []; @@ -22,11 +22,11 @@ public function setParam(array $params = []) public function getDefaultConnector(array $params = []) { - assert(!empty(getenv('FINXLOG_ELASTICO_PARAM'))); - assert(!empty(json_decode(getenv('FINXLOG_ELASTICO_PARAM')))); + assert(!empty(getenv('FINXLOG_ELASTICA_PARAM'))); + assert(!empty(json_decode(getenv('FINXLOG_ELASTICA_PARAM')))); return new \Elastica\Client( - $this->param + json_decode(getenv('FINXLOG_ELASTICO_PARAM'), true) + $this->param + json_decode(getenv('FINXLOG_ELASTICA_PARAM'), true) ); } } \ No newline at end of file diff --git a/src/Classes/Module/Import/AbsQuotation.php b/src/Classes/Module/ImportQuotation/AbsQuotation.php similarity index 91% rename from src/Classes/Module/Import/AbsQuotation.php rename to src/Classes/Module/ImportQuotation/AbsQuotation.php index 662c47e..faca8d6 100644 --- a/src/Classes/Module/Import/AbsQuotation.php +++ b/src/Classes/Module/ImportQuotation/AbsQuotation.php @@ -1,8 +1,8 @@ getModelQuotation() ->save( - Import\Source\Telnet::getFromRaw($string) + ImportQuotation\Source\Telnet::getFromRaw($string) ); return $this; diff --git a/src/Classes/Module/Import/LoadQuotation.php b/src/Classes/Module/ImportQuotation/LoadQuotation.php similarity index 95% rename from src/Classes/Module/Import/LoadQuotation.php rename to src/Classes/Module/ImportQuotation/LoadQuotation.php index d5ce693..09c2967 100644 --- a/src/Classes/Module/Import/LoadQuotation.php +++ b/src/Classes/Module/ImportQuotation/LoadQuotation.php @@ -1,5 +1,5 @@ getConnector() ->getQuotation(); - } catch (\Exception $e) { + } catch (\Throwable $e) { Logger::log()->debug('-'); Logger::error('LoadQuotation getQuotation error', $e); } + if ($import) { try { $this->saveJob($import); diff --git a/src/Classes/Module/Import/SaveQuotation.php b/src/Classes/Module/ImportQuotation/SaveQuotation.php similarity index 93% rename from src/Classes/Module/Import/SaveQuotation.php rename to src/Classes/Module/ImportQuotation/SaveQuotation.php index a7659c3..e3cb97b 100644 --- a/src/Classes/Module/Import/SaveQuotation.php +++ b/src/Classes/Module/ImportQuotation/SaveQuotation.php @@ -1,5 +1,5 @@ getQueueConnector() ->delete($queue_job); - } catch (\Exception $e) { + } catch (\Throwable $e) { Logger::log()->debug('-'); Logger::error('SaveQuotation exception', $e); $this->failQueu($queue_job); diff --git a/src/Classes/Module/Import/Source/AbsSource.php b/src/Classes/Module/ImportQuotation/Source/AbsSource.php similarity index 90% rename from src/Classes/Module/Import/Source/AbsSource.php rename to src/Classes/Module/ImportQuotation/Source/AbsSource.php index f678940..73bf12f 100644 --- a/src/Classes/Module/Import/Source/AbsSource.php +++ b/src/Classes/Module/ImportQuotation/Source/AbsSource.php @@ -1,10 +1,11 @@ 'MSG', 'T' => '2001-01-01', @@ -52,9 +53,9 @@ public static function filter(array $result) public static function prepare(array $result) { - assert(!empty($result['T'])); - $result['T'] = static::getTime($result['T']); + $result['B'] = (float) $result['B']; + assert(!empty($result['B'])); return $result; } diff --git a/src/Classes/Module/Import/Source/Telnet.php b/src/Classes/Module/ImportQuotation/Source/Telnet.php similarity index 95% rename from src/Classes/Module/Import/Source/Telnet.php rename to src/Classes/Module/ImportQuotation/Source/Telnet.php index ab993b2..1312b9a 100644 --- a/src/Classes/Module/Import/Source/Telnet.php +++ b/src/Classes/Module/ImportQuotation/Source/Telnet.php @@ -1,5 +1,5 @@ get_class($context), 'trace' =>$context->getTraceAsString() diff --git a/tests/ConnectorQuotationTest.php b/tests/ConnectorQuotationTest.php index 58b9188..3e08ff4 100644 --- a/tests/ConnectorQuotationTest.php +++ b/tests/ConnectorQuotationTest.php @@ -16,11 +16,12 @@ public function getApp() return $this->app; } - public function test_connector() + public function test_basic() { $this->assertTrue($this->getApp()->getDefaultConnector() instanceof \Socket\Raw\Socket); $this->assertTrue($this->getApp()->getConnector() instanceof \Socket\Raw\Socket); } + public function test_read() { $this->assertTrue(strlen($this->getApp()->read()) > 0); @@ -34,7 +35,7 @@ public function test_read() 'wrong result:' . $result ); - $quotation = \FinXLog\Module\Import\Source\Telnet::getFromRaw($result); + $quotation = \FinXLog\Module\ImportQuotation\Source\Telnet::getFromRaw($result); $this->assertTrue(is_array($quotation)); $this->assertTrue(count($quotation) == 3); $this->assertTrue($quotation === $quotation + ['T' => 1, 'B'=>2, 'S'=>3]); diff --git a/tests/ModelQuotationAggTest.php b/tests/ModelQuotationAggTest.php new file mode 100644 index 0000000..ff2960d --- /dev/null +++ b/tests/ModelQuotationAggTest.php @@ -0,0 +1,42 @@ +app) { + $this->app = new \FinXLog\Model\QuotationAgg(); + } + + return $this->app; + } + + public function test_basic() + { + $this->assertTrue($this->getApp() instanceof \FinXLog\Model\AbsModel); + $this->assertTrue($this->getApp()->getIndex() == 'quotation'); + $this->assertTrue($this->getApp()->getType() == 'quotation'); + } + + public function test_query() + { + $this->assertTrue($this->getApp()->getDojiQuery('q') instanceof \Elastica\Query); + $query = $this->getApp()->getDojiQuery('q')->toArray(); + $this->assertTrue(is_array($query)); + $this->assertTrue(!empty($query['query']['bool']['must'][0]['query_string']['default_field'])); + + } + + public function test_dogi() + { + try { + $this->assertTrue(is_array($this->getApp()->getDoji('BTCUSD'))); + } catch (\Elastica\Exception\ConnectionException $e) { + //connection error + } catch (Throwable $e) { + $this->assertTrue(false, '!getDoji'); + } + } +} diff --git a/tests/ModelQuotationTest.php b/tests/ModelQuotationTest.php new file mode 100644 index 0000000..a55bae0 --- /dev/null +++ b/tests/ModelQuotationTest.php @@ -0,0 +1,28 @@ +app) { + $this->app = new \FinXLog\Model\Quotation(); + } + + return $this->app; + } + + public function test_basic() + { + $this->assertTrue($this->getApp() instanceof \FinXLog\Model\AbsModel); + $this->assertTrue($this->getApp()->getDefaultConnector() instanceof \FinXLog\Module\Connector\Elastica); + $this->assertTrue($this->getApp()->getConnector() instanceof \FinXLog\Module\Connector\Elastica); + $this->assertTrue($this->getApp()->getDb() instanceof \Elastica\Client); + $this->assertTrue($this->getApp()->getElasticIndex() instanceof \Elastica\Index); + $this->assertTrue($this->getApp()->getPreparedDocument('string') instanceof \Elastica\Document); + + $this->assertTrue($this->getApp()->getIndex() == 'quotation'); + $this->assertTrue($this->getApp()->getType() == 'quotation'); + } +} From 0e07ffe07c1908c2795fec5f75fa5889c5b2dcc3 Mon Sep 17 00:00:00 2001 From: bag Date: Tue, 14 Jun 2016 08:23:44 +0300 Subject: [PATCH 07/23] add websocket: ratchet, add blank html, add nodel->getQuotation --- .env.example | 16 +- command/daemon/quotation_amqp2ws.php | 11 + command/daemon/quotation_exchange2amqp.sh | 8 +- command/daemon/quotation_websocket.php | 6 + command/daemon/quotation_ws2client.php | 6 + composer.json | 4 +- config/dbg.php | 3 +- src/Classes/Model/AbsElasticaModel.php | 12 + src/Classes/Model/Quotation.php | 29 ++ src/Classes/Model/QuotationAgg.php | 22 +- .../Module/ClientQueue/AbsQuotation.php | 96 ++++++ .../Module/ClientQueue/SearchQuotation.php | 73 ++++ .../Module/ImportQuotation/AbsQuotation.php | 4 +- .../Module/ImportQuotation/LoadQuotation.php | 4 +- .../ImportQuotation/Source/AbsSource.php | 2 +- .../Module/Ratchet/QuotationServer.php | 44 +++ .../Ratchet/QuotationWebSocketDelivery.php | 321 ++++++++++++++++++ www/index.html | 89 +++++ 18 files changed, 733 insertions(+), 17 deletions(-) create mode 100755 command/daemon/quotation_amqp2ws.php create mode 100755 command/daemon/quotation_websocket.php create mode 100755 command/daemon/quotation_ws2client.php create mode 100644 src/Classes/Module/ClientQueue/AbsQuotation.php create mode 100644 src/Classes/Module/ClientQueue/SearchQuotation.php create mode 100644 src/Classes/Module/Ratchet/QuotationServer.php create mode 100644 src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php create mode 100644 www/index.html diff --git a/.env.example b/.env.example index 36c4afb..f341e17 100644 --- a/.env.example +++ b/.env.example @@ -7,10 +7,18 @@ FINXLOG_QUOTATION_SERVER_PORT=10000 FINXLOG_AMQP_SERVER_ADDRESS=127.0.0.1 #FINXLOG_AMQP_SERVER_PORT= -FINXLOG_AMQP_TUBE_QUOTATION=finxlog_quotation -FINXLOG_AMQP_TUBE_QUOTATION_FAIL=finxlog_quotation_fail +FINXLOG_AMQP_TUBE_IMPORT=finxlog_quotation +FINXLOG_AMQP_TUBE_IMPORT_FAIL=finxlog_quotation_fail +FINXLOG_AMQP_TUBE_WS=finxlog_ws FINXLOG_ELASTICA_PARAM={"host":"localhost"} -FINXLOG_FILTER_OTHER=1 +#FINXLOG_ELASTICA_LOG_PARAM={"host":"localhost"} -#FINXLOG_ELASTICA_LOG_PARAM={"host":"localhost"} \ No newline at end of file +FINXLOG_IMPORT_FILTER_OTHER=1 + +FINXLOG_WEBSOCKET_PORT=8080 +FINXLOG_WEBSOCKET_HOST=localhost + +#this options for ignore fake service data. or send sign after connection +FINXLOG_WEBSOCKET_SERVICE_PATH=/sEcReT_SeRvIcE_pAtH +FINXLOG_WEBSOCKET_SERVICE_FILTER_ADDR_REGEXP=^127\. diff --git a/command/daemon/quotation_amqp2ws.php b/command/daemon/quotation_amqp2ws.php new file mode 100755 index 0000000..1c106fd --- /dev/null +++ b/command/daemon/quotation_amqp2ws.php @@ -0,0 +1,11 @@ +#!/usr/bin/php +run(); diff --git a/command/daemon/quotation_exchange2amqp.sh b/command/daemon/quotation_exchange2amqp.sh index 01f5b00..9b2076e 100755 --- a/command/daemon/quotation_exchange2amqp.sh +++ b/command/daemon/quotation_exchange2amqp.sh @@ -3,10 +3,12 @@ FINXLOG_ROOT_DIR=`dirname $0`/../../; #import .env export $(cat $FINXLOG_ROOT_DIR.env | grep -P '^FINXLOG_(QUOTATION|AMQP)') -#export | grep FIN +# check: +# export | grep FIN # netcat > amqp while : do - netcat $FINXLOG_QUOTATION_SERVER_ADDRESS $FINXLOG_QUOTATION_SERVER_PORT | xargs -I {} beanstool put -t=$FINXLOG_AMQP_TUBE_QUOTATION -b {} -done \ No newline at end of file + netcat $FINXLOG_QUOTATION_SERVER_ADDRESS $FINXLOG_QUOTATION_SERVER_PORT \ + | xargs -I {} beanstool put -t=$FINXLOG_AMQP_TUBE_IMPORT -b {} +done diff --git a/command/daemon/quotation_websocket.php b/command/daemon/quotation_websocket.php new file mode 100755 index 0000000..634103e --- /dev/null +++ b/command/daemon/quotation_websocket.php @@ -0,0 +1,6 @@ +#!/usr/bin/php +run(); \ No newline at end of file diff --git a/command/daemon/quotation_ws2client.php b/command/daemon/quotation_ws2client.php new file mode 100755 index 0000000..634103e --- /dev/null +++ b/command/daemon/quotation_ws2client.php @@ -0,0 +1,6 @@ +#!/usr/bin/php +run(); \ No newline at end of file diff --git a/composer.json b/composer.json index a394b16..b43d8f6 100644 --- a/composer.json +++ b/composer.json @@ -15,7 +15,9 @@ "clue/socket-raw": "~1.2", "pda/pheanstalk": "^3.1", "ruflin/Elastica": "^3.2", - "monolog/monolog": "^1.19" + "monolog/monolog": "^1.19", + "cboden/ratchet": "^0.3.5", + "ratchet/pawl": "^0.2.2" }, "require-dev": { "phpunit/phpunit": "4.7.*" diff --git a/config/dbg.php b/config/dbg.php index d1db1cc..48f38ec 100644 --- a/config/dbg.php +++ b/config/dbg.php @@ -25,8 +25,7 @@ function ($string = null) { set_error_handler( function ($code, $string, $file, $line, $context = []) { - \FinXLog\Module\Logger::log() - ->warning($string . ";code:$code at $file:$line"); + \FinXLog\Module\Logger::error($string . ";code:$code at $file:$line", $context); return true; } diff --git a/src/Classes/Model/AbsElasticaModel.php b/src/Classes/Model/AbsElasticaModel.php index 821ba03..c2b7800 100644 --- a/src/Classes/Model/AbsElasticaModel.php +++ b/src/Classes/Model/AbsElasticaModel.php @@ -1,6 +1,7 @@ getResponse($query) + ->getDocuments(); + } + } \ No newline at end of file diff --git a/src/Classes/Model/Quotation.php b/src/Classes/Model/Quotation.php index 8bbd4c0..586f984 100644 --- a/src/Classes/Model/Quotation.php +++ b/src/Classes/Model/Quotation.php @@ -1,5 +1,6 @@ query_quotations, true); + if (!$quotation) { + $query['query']['bool']['must'] = []; + } else { + $query['query']['bool']['must'][0]['query_string']['query'] = $quotation; + } + return $this->getDocuments(new Query($query)); + } } \ No newline at end of file diff --git a/src/Classes/Model/QuotationAgg.php b/src/Classes/Model/QuotationAgg.php index 7d5b7ff..086a16c 100644 --- a/src/Classes/Model/QuotationAgg.php +++ b/src/Classes/Model/QuotationAgg.php @@ -6,6 +6,18 @@ class QuotationAgg extends Quotation { + /** + * @todo period deep + * @var array + */ + protected $agg_period = [ + 'M1' => '60', + 'M5' => '300', + 'H1' => '3600', + 'D1' => '86400', + 'W1' => '604800', + ]; + /* * more test, any first, not avg first "first":{"top_hits":{"size": 1,"sort":[{"T": {"order": "asc"}}]}} @@ -90,7 +102,7 @@ class QuotationAgg extends Quotation */ public function getDoji($subject, $interval = 'day') { - return $this->getResult( + return $this->getAggregations( $this->getDojiQuery( $subject, $interval @@ -103,7 +115,7 @@ public function getDoji($subject, $interval = 'day') * @param Query $query * @return array */ - public function getResult(Query $query) + public function getAggregations(Query $query) { return current( //1st agg name is not important $this->getResponse($query) @@ -120,4 +132,10 @@ public function getDojiQuery($subject, $interval = 'day') return new Query($query); } + + public function getAggPeriod() + { + return $this->agg_period; + } + } diff --git a/src/Classes/Module/ClientQueue/AbsQuotation.php b/src/Classes/Module/ClientQueue/AbsQuotation.php new file mode 100644 index 0000000..4dd882f --- /dev/null +++ b/src/Classes/Module/ClientQueue/AbsQuotation.php @@ -0,0 +1,96 @@ +ws_connector = $ws_connector; + + return $this; + } + public function getWsConnector() + { + if (!$this->ws_connector) { + $this->ws_connector = $this->getDefaultWsConnector(); + } + + return $this->ws_connector; + } + + public function addWsMessage($message) + { + $this->getWsConnector() + ->then( + function($conn) use ($message) { + $conn->send($message); + }, + function ($e) { + Logger::error( + "WS Service: Could not connect: {$e->getMessage()}", + [ + 'e' => $e + ] + ); + } + ); + } + public function getDefaultWsConnector() + { + return \Ratchet\Client\connect( + 'ws://' . getenv('FINXLOG_WEBSOCKET_HOST') + . ':' . getenv('FINXLOG_WEBSOCKET_PORT') + . getenv('FINXLOG_WEBSOCKET_SERVICE_PATH'), + ['protocol1', 'subprotocol2'], + ['Origin' => 'http://localhost'] + ); + } + + + public function getModelQuotation() + { + if (!$this->model_quotation) { + $this->model_quotation = new Model\QuotationAgg(); + } + + return $this->model_quotation; + } + + public function getDefaultQueueConnector() + { + return new Connector\Queue( + getenv('FINXLOG_AMQP_TUBE_WS') + ); + } + + public function getFailQueueConnector() + { + //ignore + return $this; + } + public function failQueu(Job $queue_job) + { + //ignore + return $this; + } + + +} \ No newline at end of file diff --git a/src/Classes/Module/ClientQueue/SearchQuotation.php b/src/Classes/Module/ClientQueue/SearchQuotation.php new file mode 100644 index 0000000..8cb3a85 --- /dev/null +++ b/src/Classes/Module/ClientQueue/SearchQuotation.php @@ -0,0 +1,73 @@ +getData(), true); + + if (!is_array($job) || empty($job['quotation'])) { + throw new Exception\WrongParams('!job with quotation'); + } + if (!empty($job['agg'])) { + $this->addWsMessage( + $this->getModelQuotation() + ->getDoji($job['quotation'], $job['agg_period']) + ); + } else { + $this->addWsMessage( + $this->getModelQuotation() + ->getQuotations( + $job['quotation'] == static::QUOTATION_ALL + ? null + : $job['quotation'] + ) + ); + } + switch($job['agg']) { + case static::QUOTATION_ALL: + + break; + default: + + break; + } + + } + public function run() + { + $this->getQueueConnector() + ->watch(); + + while (true) { + $queue_job = $this->getQueueConnector()->reserve(); + try { + $this->makeJob($queue_job); + Logger::log()->debug('+'); + } catch (\Throwable $e) { + Logger::log()->debug('-'); + Logger::error('SaveQuotation exception', ['e' => $e, 'job' => $queue_job]); + } + //try few times + $this->getQueueConnector() + ->delete($queue_job); + } + + return $this; + } +} \ No newline at end of file diff --git a/src/Classes/Module/ImportQuotation/AbsQuotation.php b/src/Classes/Module/ImportQuotation/AbsQuotation.php index faca8d6..69283e2 100644 --- a/src/Classes/Module/ImportQuotation/AbsQuotation.php +++ b/src/Classes/Module/ImportQuotation/AbsQuotation.php @@ -23,7 +23,7 @@ public function getFailQueueConnector() { if (!$this->fail_queue_connector) { $this->fail_queue_connector = new Connector\Queue( - getenv('FINXLOG_AMQP_TUBE_QUOTATION_FAIL') + getenv('FINXLOG_AMQP_TUBE_IMPORT_FAIL') ); } @@ -42,7 +42,7 @@ public function getModelQuotation() public function getDefaultQueueConnector() { return new Connector\Queue( - getenv('FINXLOG_AMQP_TUBE_QUOTATION') + getenv('FINXLOG_AMQP_TUBE_IMPORT') ); } diff --git a/src/Classes/Module/ImportQuotation/LoadQuotation.php b/src/Classes/Module/ImportQuotation/LoadQuotation.php index 09c2967..50aa57f 100644 --- a/src/Classes/Module/ImportQuotation/LoadQuotation.php +++ b/src/Classes/Module/ImportQuotation/LoadQuotation.php @@ -42,7 +42,7 @@ public function run($limit = null) if ($import) { try { - $this->saveJob($import); + $this->addJob($import); Logger::log()->debug('+'); } catch (\Exception $e) { Logger::log()->debug('-'); @@ -54,7 +54,7 @@ public function run($limit = null) return $this; } - public function saveJob($string) + public function addJob($string) { if ($this->work_with_amqp) { if ($this->getQueueConnector()) { diff --git a/src/Classes/Module/ImportQuotation/Source/AbsSource.php b/src/Classes/Module/ImportQuotation/Source/AbsSource.php index 73bf12f..d3ad7fa 100644 --- a/src/Classes/Module/ImportQuotation/Source/AbsSource.php +++ b/src/Classes/Module/ImportQuotation/Source/AbsSource.php @@ -41,7 +41,7 @@ public static function validate(array $result) public static function filter(array $result) { - if (getenv('FINXLOG_FILTER_OTHER')) { + if (getenv('FINXLOG_IMPORT_FILTER_OTHER')) { $result = array_diff_key( $result, static::getValidQuotation() diff --git a/src/Classes/Module/Ratchet/QuotationServer.php b/src/Classes/Module/Ratchet/QuotationServer.php new file mode 100644 index 0000000..b6c1e6c --- /dev/null +++ b/src/Classes/Module/Ratchet/QuotationServer.php @@ -0,0 +1,44 @@ +io_server) { + assert(getenv('FINXLOG_WEBSOCKET_PORT') > 0); + + $this->io_server = IoServer::factory( + new HttpServer( + new WsServer( + new QuotationWebSocketDelivery() + ) + ), + getenv('FINXLOG_WEBSOCKET_PORT') + ); + } + + return $this->io_server; + } + + public function setIoServer(IoServer $io_server) + { + $this->io_server = $io_server; + + return $this; + } + + public function run() + { + $this->getIoServer()->run(); + } +} diff --git a/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php b/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php new file mode 100644 index 0000000..0a44187 --- /dev/null +++ b/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php @@ -0,0 +1,321 @@ +clients = new \SplObjectStorage; + + } + + /** + * is service => allow incoming + * @param ConnectionInterface $conn + * @return bool + */ + protected function isService(ConnectionInterface $conn, $msg = null) + { + if ( + $conn->resourceId + && isset($this->service_by_resource[$conn->resourceId]) + ) { + return $this->service_by_resource[$conn->resourceId]; + } + assert(strlen( + getenv('FINXLOG_WEBSOCKET_SERVICE_PATH') + . getenv('FINXLOG_WEBSOCKET_SERVICE_FILTER_ADDR_REGEXP') + )); + + $is_service = true; + + if (strlen(getenv('FINXLOG_WEBSOCKET_SERVICE_FILTER_ADDR_REGEXP'))) { + $is_service = (bool)preg_match( + '~' . getenv('FINXLOG_WEBSOCKET_SERVICE_FILTER_ADDR_REGEXP') . '~iu', + $conn->remoteAddress + ); + } + + try { + if (getenv('FINXLOG_WEBSOCKET_SERVICE_PATH')) { + $is_service = ( + $is_service + && $conn instanceof RFC6455\Connection + && $conn->WebSocket->request + && strstr( + $conn->WebSocket->request->getPath(), + getenv('FINXLOG_WEBSOCKET_SERVICE_PATH'), + 1 + ) + ); + } + } catch (\Throwable $e) { + Logger::error( + $e->getMessage(), + ['e' => $e] + ); + + $is_service = false; + } + + return $this->service_by_resource[$conn->resourceId] = $is_service; + } + + public function onOpen(ConnectionInterface $conn) + { + if (!$this->isService($conn)) { + //waiting for manual subscribe + //$this->clients['quotation'][static::QUOTATION_ALL]->attach($conn); + } + + Logger::log()->debug( + "WS: New " + . ( + $this->isService($conn) + ? 'service' + : 'client' + ) + . " connection: {$conn->resourceId}", + [ + 'is_service' => $this->isService($conn), + 'conn' => $conn + ] + ); + } + + public function getPreparedMessage(ConnectionInterface $conn, $msg) + { + $message = json_decode($msg, true); + if ( + empty($message) + || !is_array($message) + || empty($message['type']) + ) { + throw new WrongParams("WS: !msg: $msg"); + } + if ( + !empty($message['quotation']) + && !preg_match('~^[\w\d_\.]+$~u', $message['quotation']) + ) { + throw (new WrongParams("WS: wrong quotation: $msg")) + ->setParams(['quotation']); + } + if ($message['type'] == 'subscribe' && empty($message['quotation'])) { + $message['quotation'] = static::QUOTATION_ALL; + } + + if ( + !empty($message['doji']) + && !preg_match('~^[\w\d_\.]+$~u', $message['doji']) + ) { + throw (new WrongParams("WS: wrong doji: $msg")) + ->setParams(['doji']); + } + + return $message; + } + + public function onMessage(ConnectionInterface $conn, $msg) + { + Logger::log()->debug( + "WS msg from #{$conn->resourceId}: {$msg}", + [ + 'conn' => $conn, + 'is_service' => $this->isService($conn), + 'conn_count' => count($this->service_by_resource), + ] + ); + + if (!$this->isService($conn) && strlen($msg) > 1000) { + Logger::log()->warning( + "WS: msg with " . round(strlen($msg) / 1000). "kb", + [ + 'conn' => $conn, + 'is_service' => $this->isService($conn), + 'conn_count' => count($this->service_by_resource), + ] + ); + } + + try { + $message = $this->getPreparedMessage($conn, $msg); + if ($this->isService($conn, $message)) { + $this->addServiceIncoming($conn, $message); + } + //allow service subscribe + $this->addClientIncoming($conn, $message); + } catch (\Throwable $e) { + Logger::log()->warning( + "WS: {$e->getMessage()} from " + . ( + $this->isService($conn) + ? 'service' + : 'client' + ) + . " {$conn->resourceId}: : {$msg}", + [ + 'e' => $e, + 'conn' => $conn, + 'is_service' => $this->isService($conn), + 'conn_count' => count($this->service_by_resource), + ] + ); + } + } + + /** + * delivery to clients + * @param ConnectionInterface $conn + * @param string $msg + */ + protected function addServiceIncoming(ConnectionInterface $conn, array $message) + { + assert($this->isService($conn)); + assert(!empty($message['type'])); + + switch ($message['type']) { + case 'status': + case 'stat': + $conn->send(json_encode([ + 'conn_count' => count($this->service_by_resource), + ])); + break; + case 'all': + foreach ($this->clients as $client) { + if (!$this->isService($client)) { + $conn->send(json_encode($message)); + } + } + break; + case 'send': + foreach ( + $this->subscribers + [$message['quotation']] + [empty($message['agg_period']) ? 0 : $message['agg_period']] + as $subscriber + ) { + $subscriber['ws']->send(json_encode($message)); + } + break; + } + } + + /** + * set client opts + * @param ConnectionInterface $conn + * @param string $msg + */ + protected function addClientIncoming(ConnectionInterface $conn, array $message) + { + assert(!empty($message['type'])); + if ($message['type'] != 'subscribe') { + return false; + } + $agg_period = 0; + if (!empty($message['doji'])) { + $all_period = (new QuotationAgg)->getAggPeriod()[strtoupper($message['doji'])]; + if (empty($all_period[strtoupper($message['doji'])])) { + throw (new WrongParams('WS: wrong doji period on subscribe')) + ->setParams(['doji']); + } + $agg_period = $all_period[strtoupper($message['doji'])]; + } + /** + * [USDEUR][3600][#123] = [ ... ]; + */ + $this->subscribers + [$message['quotation']] + [$agg_period] + [$conn->resourceId] + = [ + 'ws' => $conn, + 'agg' => !empty($message['doji']) ? 'doji' : null, + ]; + + //load previous period + $this->addJob([ + 'quotation' => $message['quotation'], + 'agg' => !empty($message['doji']) ? 'doji' : null, + 'agg_period' => !empty($message['doji']) ? $message['doji'] : null, + ]); + + //@todo add queue + } + + public function onClose(ConnectionInterface $conn) + { + Logger::log()->debug( + "WS:Disconnect with #{$conn->resourceId}", + [ + 'conn' => $conn, + ] + ); + + unset($this->service_by_resource[$conn->resourceId]); + + foreach ($this->subscribers as $k1 => $v1) { + foreach ($v1 as $k2=>$v2) { + unset($this->subscribers[$k1][$k2][$conn->resourceId]); + } + $this->subscribers[$k1] = array_filter($this->subscribers[$k1]); + } + $this->subscribers = array_filter($this->subscribers); + $this->clients->detach($conn); + + } + + public function onError(ConnectionInterface $conn, \Exception $e) + { + Logger::log()->debug( + "WS: " .get_class($e) . ": {$e->getMessage()} with #{$conn->resourceId}", + [ + //'this', $this, + 'e' => $e, + 'conn' => $conn + ] + ); + $conn->close(); + } + + public function getDefaultQueueConnector() + { + return new Connector\Queue( + getenv('FINXLOG_AMQP_TUBE_WS') + ); + } + + public function addJob(array $array) + { + $this->getQueueConnector() + ->put($array); + + return $this; + } +} diff --git a/www/index.html b/www/index.html new file mode 100644 index 0000000..f51edf6 --- /dev/null +++ b/www/index.html @@ -0,0 +1,89 @@ + + + + + + + +
+ + + From df01621cf275fa5a7b8940ed35537be5ec8378d2 Mon Sep 17 00:00:00 2001 From: BAGArt Date: Wed, 15 Jun 2016 13:35:05 +0300 Subject: [PATCH 08/23] Update readme.md readme: complite --- readme.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/readme.md b/readme.md index b83f596..4c252b9 100644 --- a/readme.md +++ b/readme.md @@ -18,10 +18,8 @@ Optional: что надо сделать - - модуль для аггрегации данных https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html - - браузерный static интерфейс с highcharts, etc + - доделать подгрузку highcharts+websocket - простое json API - - испрользовать те же очереди для доставки клиентам(websocket, socket.io) параллельно с сохранением в БД Optional: - sql db From abd3264ab516aabeecfeb52f60d062c585b69fd4 Mon Sep 17 00:00:00 2001 From: bag Date: Sat, 18 Jun 2016 05:24:44 +0300 Subject: [PATCH 09/23] add websocket: ratchet server+client blank html add model->getQuotation path managment logger readme --- .env.example | 53 ++++- .gitignore | 1 + .../daemon/{ => import}/quotation_amqp2db.php | 2 +- .../{ => import}/quotation_exchange2amqp.php | 2 +- .../{ => import}/quotation_exchange2amqp.sh | 2 +- .../{ => import}/quotation_exchange2db.php | 2 +- command/daemon/import/readme.md | 6 + command/daemon/quotation_amqp2ws.php | 11 - command/daemon/quotation_websocket.php | 6 - command/daemon/quotation_ws2client.php | 6 - command/daemon/ws/quotation_amqp2ws.php | 6 + command/daemon/ws/quotation_ws2client.php | 6 + command/daemon/ws/readme.md | 37 +++ command/setup/web_config_build.php | 27 +++ command/test/add_amqp_ws_subscribe.php | 17 ++ command/test/add_amqp_ws_subscribe.sh | 8 + command/test/add_ws_msg_service.php | 13 ++ composer.json | 5 +- readme.md | 59 +++-- src/Classes/Iface/WsConnector.php | 9 + src/Classes/Model/Quotation.php | 1 + src/Classes/Model/QuotationAgg.php | 5 +- .../Module/ClientQueue/AbsQuotation.php | 62 +++-- .../Module/ClientQueue/SearchQuotation.php | 52 ++--- .../Module/Connector/RatchetClient.php | 160 +++++++++++++ .../Module/ImportQuotation/SaveQuotation.php | 13 +- src/Classes/Module/Logger.php | 82 +++++-- .../Module/Ratchet/QuotationServer.php | 7 +- .../Ratchet/QuotationWebSocketDelivery.php | 211 +++++++++--------- src/Classes/Module/Ratchet/Subscribers.php | 106 +++++++++ www/index.html | 80 +------ www/js/.gitkeep | 0 www/js/app.js | 140 ++++++++++++ 33 files changed, 885 insertions(+), 312 deletions(-) rename command/daemon/{ => import}/quotation_amqp2db.php (86%) rename command/daemon/{ => import}/quotation_exchange2amqp.php (79%) rename command/daemon/{ => import}/quotation_exchange2amqp.sh (88%) rename command/daemon/{ => import}/quotation_exchange2db.php (82%) create mode 100644 command/daemon/import/readme.md delete mode 100755 command/daemon/quotation_amqp2ws.php delete mode 100755 command/daemon/quotation_websocket.php delete mode 100755 command/daemon/quotation_ws2client.php create mode 100755 command/daemon/ws/quotation_amqp2ws.php create mode 100755 command/daemon/ws/quotation_ws2client.php create mode 100644 command/daemon/ws/readme.md create mode 100755 command/setup/web_config_build.php create mode 100755 command/test/add_amqp_ws_subscribe.php create mode 100755 command/test/add_amqp_ws_subscribe.sh create mode 100755 command/test/add_ws_msg_service.php create mode 100644 src/Classes/Iface/WsConnector.php create mode 100644 src/Classes/Module/Connector/RatchetClient.php create mode 100644 src/Classes/Module/Ratchet/Subscribers.php create mode 100644 www/js/.gitkeep create mode 100644 www/js/app.js diff --git a/.env.example b/.env.example index f341e17..6968c4f 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,7 @@ PROJECT_NAME=FonXLog -FINXLOG_DEBUG=0 +#Monolog\Logger::DEBUG = 100 +FINXLOG_DEBUG=100 +FINXLOG_DOMAIN=localhost FINXLOG_QUOTATION_SERVER_ADDRESS=178.62.145.164 #FINXLOG_QUOTATION_SERVER_ADDRESS=198.211.118.180 @@ -14,11 +16,50 @@ FINXLOG_AMQP_TUBE_WS=finxlog_ws FINXLOG_ELASTICA_PARAM={"host":"localhost"} #FINXLOG_ELASTICA_LOG_PARAM={"host":"localhost"} -FINXLOG_IMPORT_FILTER_OTHER=1 +FINXLOG_IMPORT_FILTER_OTHER=0 -FINXLOG_WEBSOCKET_PORT=8080 -FINXLOG_WEBSOCKET_HOST=localhost +FINXLOG_ENV_JS="PROJECT_NAME FINXLOG_DOMAIN FINXLOG_WEBSOCKET_EXTARNAL_HOST FINXLOG_WEBSOCKET_EXTARNAL_PORT FINXLOG_WEBSOCKET_EXTARNAL_PATH FINXLOG_DEBUG" -#this options for ignore fake service data. or send sign after connection -FINXLOG_WEBSOCKET_SERVICE_PATH=/sEcReT_SeRvIcE_pAtH + + +##### WEBSOCKET BLOCK +#please, always use +# 1. not LISTEN public interface (use 127.0.0.1 or internal) +# 2. setup http proxy on vhost to websocket. nginx: http://nginx.org/ru/docs/http/websocket.html +# //DOMAIN/ws => ws://LISTEN_INTERFACE:LISTEN_PORT/____NOT____SERVICE_PATH +# //localhost/ws => ws://127.0.0.1:8080/ws + +##### -------------------------------- +###DEVELOPER MODE. is just for quick start. do not use +#is direct access for send any data to any users +FINXLOG_WEBSOCKET_LISTEN_INTERFACE=0.0.0.0 +FINXLOG_WEBSOCKET_EXTARNAL_PORT=8080 FINXLOG_WEBSOCKET_SERVICE_FILTER_ADDR_REGEXP=^127\. + +###PRODUCTION MODE: +#FINXLOG_WEBSOCKET_LISTEN_INTERFACE=127.0.0.1 +#default EXTARNAL_PORT=80 + +###PRODUCTION CLOUD MODE with 10.0.3.0 intranet +#FINXLOG_WEBSOCKET_LISTEN_INTERFACE=10.0.0.0 +#FINXLOG_WEBSOCKET_INTERNAL_HOST=10.0.3.x +#default EXTARNAL_PORT=80 +##### -------------------------------- + + + + +##### WEBSOCKET TYPICAL BLOCK +#required for internal service +FINXLOG_WEBSOCKET_LISTEN_PORT=8080 + +#markers for proxy: path or domain +FINXLOG_WEBSOCKET_EXTARNAL_PATH=/ws +#default: FINXLOG_DOMAIN +#FINXLOG_WEBSOCKET_EXTARNAL_HOST=localhost + +#if empty(FILTER_ADDR_REGEXP) and proxy not acceptable. randomize it: for same secure +FINXLOG_WEBSOCKET_SERVICE_PATH=/service + +#one of: Ratchet or ElephantIO. just for performance test +#FINXLOG_WEBSOCKET_SERVICE_LIB=Ratchet diff --git a/.gitignore b/.gitignore index 2206342..ef4215d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ vendor/ composer.lock .env +www/js/config_autobuild.js diff --git a/command/daemon/quotation_amqp2db.php b/command/daemon/import/quotation_amqp2db.php similarity index 86% rename from command/daemon/quotation_amqp2db.php rename to command/daemon/import/quotation_amqp2db.php index 28a84c0..0c31915 100755 --- a/command/daemon/quotation_amqp2db.php +++ b/command/daemon/import/quotation_amqp2db.php @@ -4,7 +4,7 @@ /** * Always on service: load quotation from source */ -require_once __DIR__ . '/../../vendor/autoload.php'; +require_once __DIR__ . '/../../../vendor/autoload.php'; $import = (new FinXLog\Module\ImportQuotation\SaveQuotation()); diff --git a/command/daemon/quotation_exchange2amqp.php b/command/daemon/import/quotation_exchange2amqp.php similarity index 79% rename from command/daemon/quotation_exchange2amqp.php rename to command/daemon/import/quotation_exchange2amqp.php index 7a6b30c..c75cef5 100755 --- a/command/daemon/quotation_exchange2amqp.php +++ b/command/daemon/import/quotation_exchange2amqp.php @@ -4,7 +4,7 @@ /** * Always on service: load quotation from source */ -require_once __DIR__ . '/../../vendor/autoload.php'; +require_once __DIR__ . '/../../../vendor/autoload.php'; $import = new FinXLog\Module\ImportQuotation\LoadQuotation(); diff --git a/command/daemon/quotation_exchange2amqp.sh b/command/daemon/import/quotation_exchange2amqp.sh similarity index 88% rename from command/daemon/quotation_exchange2amqp.sh rename to command/daemon/import/quotation_exchange2amqp.sh index 9b2076e..f639348 100755 --- a/command/daemon/quotation_exchange2amqp.sh +++ b/command/daemon/import/quotation_exchange2amqp.sh @@ -1,5 +1,5 @@ #!/bin/bash -FINXLOG_ROOT_DIR=`dirname $0`/../../; +FINXLOG_ROOT_DIR=`dirname $0`/../../../; #import .env export $(cat $FINXLOG_ROOT_DIR.env | grep -P '^FINXLOG_(QUOTATION|AMQP)') diff --git a/command/daemon/quotation_exchange2db.php b/command/daemon/import/quotation_exchange2db.php similarity index 82% rename from command/daemon/quotation_exchange2db.php rename to command/daemon/import/quotation_exchange2db.php index d2f48cc..5e3f32c 100755 --- a/command/daemon/quotation_exchange2db.php +++ b/command/daemon/import/quotation_exchange2db.php @@ -4,7 +4,7 @@ /** * Always on service: load quotation from source */ -require_once __DIR__ . '/../../vendor/autoload.php'; +require_once __DIR__ . '/../../../vendor/autoload.php'; $import = new FinXLog\Module\ImportQuotation\LoadQuotation(); diff --git a/command/daemon/import/readme.md b/command/daemon/import/readme.md new file mode 100644 index 0000000..e94c7cc --- /dev/null +++ b/command/daemon/import/readme.md @@ -0,0 +1,6 @@ +# Description +this daemons for import exchange quotations + +daemon: ex2db +or +daemon: ex2amqp->AMQP->amqp2db \ No newline at end of file diff --git a/command/daemon/quotation_amqp2ws.php b/command/daemon/quotation_amqp2ws.php deleted file mode 100755 index 1c106fd..0000000 --- a/command/daemon/quotation_amqp2ws.php +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/php -run(); diff --git a/command/daemon/quotation_websocket.php b/command/daemon/quotation_websocket.php deleted file mode 100755 index 634103e..0000000 --- a/command/daemon/quotation_websocket.php +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/php -run(); \ No newline at end of file diff --git a/command/daemon/quotation_ws2client.php b/command/daemon/quotation_ws2client.php deleted file mode 100755 index 634103e..0000000 --- a/command/daemon/quotation_ws2client.php +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/php -run(); \ No newline at end of file diff --git a/command/daemon/ws/quotation_amqp2ws.php b/command/daemon/ws/quotation_amqp2ws.php new file mode 100755 index 0000000..a6da486 --- /dev/null +++ b/command/daemon/ws/quotation_amqp2ws.php @@ -0,0 +1,6 @@ +#!/usr/bin/php +run(); diff --git a/command/daemon/ws/quotation_ws2client.php b/command/daemon/ws/quotation_ws2client.php new file mode 100755 index 0000000..551048a --- /dev/null +++ b/command/daemon/ws/quotation_ws2client.php @@ -0,0 +1,6 @@ +#!/usr/bin/php +run(); \ No newline at end of file diff --git a/command/daemon/ws/readme.md b/command/daemon/ws/readme.md new file mode 100644 index 0000000..0c23e36 --- /dev/null +++ b/command/daemon/ws/readme.md @@ -0,0 +1,37 @@ +# Description +this daemons to work with Browser WebSocket request from JS_APP + +## 1st part + +get quotations list(or agg like doji) + +JS_App +->WebSocket (client) +->daemon: ws2client (listen) +->AMQP +->daemon: amqp2ws (listen) +->Elasticsearch +//back + ->WebSocket (service) + ->daemon: ws2client (listen) + ->WebSocket (all) + ->JS_App + ->table + ->JS_HighCharts + + + +@todo: +2nd part +daemon: +import/ex2amqp (listen) +->AMQP +->import/amqp2db + ->2db + ->WebSocket (service) + ->daemon: ws2client (listen) + ->WebSocket (update) + ->JS_App + ->table + ->JS_HighCharts + \ No newline at end of file diff --git a/command/setup/web_config_build.php b/command/setup/web_config_build.php new file mode 100755 index 0000000..3c2f734 --- /dev/null +++ b/command/setup/web_config_build.php @@ -0,0 +1,27 @@ +#!/usr/bin/php +put([ + 'quotation' => 'BTCUSD', + 'agg' => null, + 'agg_period' => null, + ]) + ->put([ + 'quotation' => 'BTCUSD', + 'agg' => 'doji', + 'agg_period' => key((new \FinXLog\Model\QuotationAgg)->getAggPeriod()), + ]); \ No newline at end of file diff --git a/command/test/add_amqp_ws_subscribe.sh b/command/test/add_amqp_ws_subscribe.sh new file mode 100755 index 0000000..8291f5e --- /dev/null +++ b/command/test/add_amqp_ws_subscribe.sh @@ -0,0 +1,8 @@ +#!/bin/bash +FINXLOG_ROOT_DIR=`dirname $0`/../../; +#import .env +export $(cat $FINXLOG_ROOT_DIR.env | grep -P '^FINXLOG_(QUOTATION|AMQP)') + +beanstool put -t=$FINXLOG_AMQP_TUBE_WS -b "{\"quotation\":\"BTCUSD\",\"agg\":null,\"agg_period\":null}" + +beanstool put -t=$FINXLOG_AMQP_TUBE_WS -b "{\"quotation\":\"BTCUSD\",\"agg\":\"doji\",\"agg_period\":\"M1\"}" \ No newline at end of file diff --git a/command/test/add_ws_msg_service.php b/command/test/add_ws_msg_service.php new file mode 100755 index 0000000..295de1a --- /dev/null +++ b/command/test/add_ws_msg_service.php @@ -0,0 +1,13 @@ +#!/usr/bin/php +addWsMessage(['type' => 'test']); +$client->addWsMessage(['type' => 'test']); +$client->addWsMessage(['type' => 'test']); +$client->addWsMessage(['type' => 'test']); + +$client->addWsMessage(' + {"type":"send","quotation":"_ALL","agg_period":null,"quotations":[{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/09 04:05:51","B":2.151},{"S":"TST","T":"2016\/06\/02 04:05:51","B":4},{"S":"BTCUSD","T":"2016\/06\/06 01:05:51","B":1.5},{"S":"BTCUSD","T":"2016\/06\/11 11:00:00","B":99},{"S":"BTCUSD","T":"2016\/06\/11 00:00:01","B":1},{"S":"BTCUSD","T":"2016\/06\/11 03:53:42","B":567.5},{"S":"TST","T":"2016\/06\/01 04:05:51","B":3.1},{"S":"BTCUSD","T":"2016\/06\/06 01:00:51","B":2},{"S":"BTCUSD","T":"2017\/06\/11 12:00:00","B":1},{"S":"BTCUSD","T":"2017\/06\/11 11:00:00","B":99},{"S":"BTCUSD","T":"2016\/06\/11 12:00:01","B":1},{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/10 04:05:51","B":1.151},{"S":"BTCUSD","T":"2015\/06\/07 04:05:51","B":2},{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/07 03:05:51","B":3.5},{"S":"TST2","T":"2016\/06\/06 04:05:51","B":5},{"S":"TST2","T":"2016\/06\/07 04:05:51","B":6},{"S":"BTCUSD","T":"2016\/06\/11 23:00:00","B":99},{"S":"BTCUSD","T":"2016\/06\/11 23:00:00","B":999},{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/08 04:05:51","B":3.1},{"S":"BTCUSD","T":"2017\/06\/07 04:05:51","B":1},{"S":"BTCUSD","T":"2016\/06\/11 12:00:00","B":1},{"S":"BTCUSD","T":"2016\/06\/11 00:00:01","B":10}]} + '); \ No newline at end of file diff --git a/composer.json b/composer.json index b43d8f6..97b274a 100644 --- a/composer.json +++ b/composer.json @@ -32,10 +32,11 @@ }, "scripts": { "post-install-cmd": [ - "cp .env.example .env" + "cp .env.example .env", + "php command/setup/web_config_build.php" ], "post-update-cmd": [ - + "php command/setup/web_config_build.php" ] } } diff --git a/readme.md b/readme.md index 4c252b9..11e58d3 100644 --- a/readme.md +++ b/readme.md @@ -15,12 +15,10 @@ Instruments: Optional: - BeanstalkD(AMQP Queue manager). reason for use: quick delivery for "any" load with single stream(bash-scrpit) - - -что надо сделать - - доделать подгрузку highcharts+websocket - - простое json API - +@todo + - highcharts on websocket + - simple json API + - Ratchet + WAMP + ZMQ Optional: - sql db - make quotation_exchange2db.sh @@ -29,7 +27,12 @@ Optional: # Install ```bash #install requirements -apt-get install php7.0 composer beanstalkd postgresql-9.5 php7.0-pgsql +sudo apt-get install php7.0 composer beanstalkd postgresql-9.5 php7.0-pgsql php7.0-mbstring + +#optional. cur. not ready +#sudo apt-get install php7.0-dev php-pear +#all questions - ENTER +#sudo pecl install event # https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-repositories.html wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - @@ -48,32 +51,48 @@ composer update Daemon for import quotation: ```bash -command/daemon/quotation_exchange2db.php +command/daemon/import/quotation_exchange2db.php ``` -## Optional: use with AMQP Queue for exchange high traffic +## Optional: import with AMQP for scaling high traffic important: direct import is more quickly, if server has free resource - +important: direct import has minimal guarantee for stable: + mem leak + elastic can go to repair node with slow insert + crush the process leads to a loss of traffic +AMQP is depend by high performance, scalable beanstalk (and opensource client) + + +## Required(curently - ): WebSocket +WebSocket is require AMQP + or need implement async elasticsearch client with guzzle or reactphp/http-client + or need implement async reactphp/child-process +not ready but simple: + AMQP+AJAX+API can work without WebSocket ```bash -#run each daemons -command/daemon/quotation_exchange2amqp.php - -#load daemons can work on other servers with fork -command/daemon/quotation_amqp2db.php -command/daemon/quotation_amqp2db.php fail +#load daemons can work on other servers with multiple fork +command/daemon/import/quotation_amqp2db.php +command/daemon/import/quotation_amqp2db.php fail ``` -## Optional BASH replacement for quotation_load.php -is direct linux-way socket to amqp pipe for high performance -source: https://github.com/src-d/beanstool + +## Default exchange IMPORT (not best choice) ```bash -command/daemon/quotation_exchange2amqp.sh +#run only one daemons +command/daemon/import/quotation_exchange2amqp.php ``` +## Optional exchange IMPORT for hight traffic BASH replacement for quotation_load.php +is direct linux-way socket to amqp pipe for high performance +source: https://github.com/src-d/beanstool ### Install beanstool ```bash wget https://github.com/src-d/beanstool/releases/download/v0.2.0/beanstool_v0.2.0_linux_amd64.tar.gz tar -xvzf beanstool_v0.2.0_linux_amd64.tar.gz sudo cp beanstool_v0.2.0_linux_amd64/beanstool /usr/local/bin/ ``` +## run +```bash +command/daemon/quotation_exchange2amqp.sh +``` diff --git a/src/Classes/Iface/WsConnector.php b/src/Classes/Iface/WsConnector.php new file mode 100644 index 0000000..2fa491b --- /dev/null +++ b/src/Classes/Iface/WsConnector.php @@ -0,0 +1,9 @@ +getDocuments(new Query($query)); } } \ No newline at end of file diff --git a/src/Classes/Model/QuotationAgg.php b/src/Classes/Model/QuotationAgg.php index 086a16c..0ce6761 100644 --- a/src/Classes/Model/QuotationAgg.php +++ b/src/Classes/Model/QuotationAgg.php @@ -128,7 +128,10 @@ public function getDojiQuery($subject, $interval = 'day') $query = json_decode($this->query_doji, true); $query['query']['bool']['must'][0]['query_string']['query'] = $subject; - $query['aggs']['date']['date_histogram']['interval'] = $interval; + $query['aggs']['date']['date_histogram']['interval'] = + isset($this->agg_period[$interval]) + ? $this->agg_period[$interval] . 's' + : $interval; return new Query($query); } diff --git a/src/Classes/Module/ClientQueue/AbsQuotation.php b/src/Classes/Module/ClientQueue/AbsQuotation.php index 4dd882f..09ef939 100644 --- a/src/Classes/Module/ClientQueue/AbsQuotation.php +++ b/src/Classes/Module/ClientQueue/AbsQuotation.php @@ -1,13 +1,14 @@ ws_connector = $ws_connector; return $this; } + + /** + * @return Iface\WsConnector + * @throws WrongParams + */ public function getWsConnector() { if (!$this->ws_connector) { @@ -36,34 +41,43 @@ public function getWsConnector() return $this->ws_connector; } + public function getDefaultWsConnector() + { + return (new Connector\RatchetClient) + ->setUrl($this->getWsUrl()); + } + public function addWsMessage($message) { - $this->getWsConnector() - ->then( - function($conn) use ($message) { - $conn->send($message); - }, - function ($e) { - Logger::error( - "WS Service: Could not connect: {$e->getMessage()}", - [ - 'e' => $e - ] - ); - } - ); + $this->getWsConnector()->send($message); + + return $this; } - public function getDefaultWsConnector() + + public function getWsUrl() { - return \Ratchet\Client\connect( - 'ws://' . getenv('FINXLOG_WEBSOCKET_HOST') - . ':' . getenv('FINXLOG_WEBSOCKET_PORT') - . getenv('FINXLOG_WEBSOCKET_SERVICE_PATH'), - ['protocol1', 'subprotocol2'], - ['Origin' => 'http://localhost'] + if (empty(getenv('FINXLOG_WEBSOCKET_LISTEN_PORT'))) { + throw new WrongParams('empty: FINXLOG_WEBSOCKET_LISTEN_PORT'); + } + $host = ( + getenv('FINXLOG_WEBSOCKET_INTERNAL_HOST') + ? getenv('FINXLOG_WEBSOCKET_INTERNAL_HOST') + : ( + getenv('FINXLOG_WEBSOCKET_LISTEN_INTERFACE') + ? getenv('FINXLOG_WEBSOCKET_LISTEN_INTERFACE') + : '127.0.0.1' + ) ); - } + return "ws://$host" + . ( + getenv('FINXLOG_WEBSOCKET_LISTEN_PORT') + ? ':' . getenv('FINXLOG_WEBSOCKET_LISTEN_PORT') + : null + ) + . getenv('FINXLOG_WEBSOCKET_SERVICE_PATH'); + + } public function getModelQuotation() { diff --git a/src/Classes/Module/ClientQueue/SearchQuotation.php b/src/Classes/Module/ClientQueue/SearchQuotation.php index 8cb3a85..9744ee2 100644 --- a/src/Classes/Module/ClientQueue/SearchQuotation.php +++ b/src/Classes/Module/ClientQueue/SearchQuotation.php @@ -1,6 +1,7 @@ addWsMessage( - $this->getModelQuotation() - ->getDoji($job['quotation'], $job['agg_period']) - ); + $quotations = $this->getModelQuotation() + ->getDoji($job['quotation'], $job['agg_period']); } else { - $this->addWsMessage( - $this->getModelQuotation() - ->getQuotations( - $job['quotation'] == static::QUOTATION_ALL - ? null - : $job['quotation'] - ) - ); - } - switch($job['agg']) { - case static::QUOTATION_ALL: - - break; - default: - - break; + $quotations = $this->getModelQuotation() + ->getQuotations( + $job['quotation'] == static::QUOTATION_ALL + ? null + : $job['quotation'] + ); + foreach ($quotations as $key => $value) { + /** + * @var $value Document + */ + $quotations[$key] = $value->getData(); + } } - + $this->addWsMessage([ + 'type' => 'send', + 'quotation' => $job['quotation'], + 'agg_period' => $job['agg_period'], + 'quotations' => $quotations, + ]); } + public function run() { $this->getQueueConnector() ->watch(); + //$this->getWsConnector()->send(['type' => 'test']); while (true) { + Logger::dbg(':amqp_wait:'); $queue_job = $this->getQueueConnector()->reserve(); + Logger::dbg(':amqp_make:'); try { $this->makeJob($queue_job); Logger::log()->debug('+'); } catch (\Throwable $e) { Logger::log()->debug('-'); - Logger::error('SaveQuotation exception', ['e' => $e, 'job' => $queue_job]); + Logger::error("SaveQuotation exception: {$e->getMessage()}", ['e' => $e, 'job' => $queue_job]); } //try few times $this->getQueueConnector() diff --git a/src/Classes/Module/Connector/RatchetClient.php b/src/Classes/Module/Connector/RatchetClient.php new file mode 100644 index 0000000..d2154a6 --- /dev/null +++ b/src/Classes/Module/Connector/RatchetClient.php @@ -0,0 +1,160 @@ + 'http://localhost' + ]; + private $subProtocols = []; + + /** + * BEFORE working with connector + * @param $url + */ + public function setUrl($url) + { + $this->url = $url; + + return $this; + } + + protected function getEventLoop() + { + if (!$this->event_loop) { + $this->event_loop = \React\EventLoop\Factory::create(); + } + + return $this->event_loop; + } + + protected function getPromise() + { + if (empty($this->url)) { + throw new WrongParams(__CLASS__ . 'use setUrl first'); + } + + if (!$this->promise) { + $connector = $this->getConnector(); + $this->promise = $connector( + $this->url, + $this->subProtocols, + $this->headers + ); + } + + return $this->promise; + } + + /** + * real work with $this->promise->then! + * @return \Ratchet\Client\Connector + */ + public function getDefaultConnector() + { + return new \Ratchet\Client\Connector($this->getEventLoop()); + } + + public function close() + { + if ($this->event_loop) { + $this->event_loop->stop(); + } + + return $this; + } + + public function send($message, callable $incomingCallback = null) + { + //force new connect + //$this->promise = null; + //$this->event_loop = null; + //$this->connector = null; + $is_first = empty($this->event_loop); + $event_loop = $this->getEventLoop(); + + $this->getPromise()->then( + function(\Ratchet\Client\WebSocket $conn) + use ($message, $event_loop, $incomingCallback) + { + if (!is_string($message)) { + $message = json_encode($message); + } + if (getenv('FINXLOG_DEBUG')) { + Logger::log()->info("AMQP2WS(service) send try:\t{$message}"); + if (empty($conn->listeners('close'))) { //new connect + $conn->on( + 'close', + function ($code = null, $reason = null) + use($incomingCallback) { + Logger::log()->info( + "AMQP2WS(service) closed: ({$code} - {$reason})" + ); + if ($incomingCallback) { + Logger::log()->info( + "AMQP2WS(service) incomingCallback: start" + ); + $incomingCallback(); + Logger::log()->info( + "AMQP2WS(service) incomingCallback: end" + ); + } + }); + //@todo check that the socket stream is empty(released) without on:event + $conn->on('messa;ge', function ($message) use ($conn) { + Logger::log()->warning( + "AMQP2WS(service) ignore message: {$message}" + ); + }); + } + } + $conn->send($message); + + //exit from EventLoop + $event_loop->stop(); + }, + function (\Throwable $e) + use ($event_loop, $message, $incomingCallback) + { + Logger::error( + "WS Service error: Could not connect: {$e->getMessage()}", + [ + 'e' => $e, + 'incomingCallback' => $incomingCallback, + ] + ); + $event_loop->stop(); + } + ); + if ($is_first) { + $event_loop->run(); + } + //push all queue + $event_loop->tick(); + + return $this; + } + +} \ No newline at end of file diff --git a/src/Classes/Module/ImportQuotation/SaveQuotation.php b/src/Classes/Module/ImportQuotation/SaveQuotation.php index e3cb97b..048f682 100644 --- a/src/Classes/Module/ImportQuotation/SaveQuotation.php +++ b/src/Classes/Module/ImportQuotation/SaveQuotation.php @@ -16,23 +16,18 @@ class SaveQuotation extends AbsQuotation { public function run($limit = null) { - $this->getQueueConnector() - ->watch(); - while ($limit === null || --$limit >= 0) { $queue_job = $this->getQueueConnector()->reserve(); try { $this->importQuotation($queue_job->getData()); Logger::log()->debug('+'); - $this->getQueueConnector() - ->delete($queue_job); + $this->getQueueConnector()->delete($queue_job); } catch (Exception\WrongImport $e) { - Logger::log()->debug('-'); + //Logger::log()->debug('-'); Logger::error('SaveQuotation WrongImport', $e); - $this->getQueueConnector() - ->delete($queue_job); + $this->getQueueConnector()->delete($queue_job); } catch (\Throwable $e) { - Logger::log()->debug('-'); + //Logger::log()->debug('-'); Logger::error('SaveQuotation exception', $e); $this->failQueu($queue_job); } diff --git a/src/Classes/Module/Logger.php b/src/Classes/Module/Logger.php index eca125d..de8726c 100644 --- a/src/Classes/Module/Logger.php +++ b/src/Classes/Module/Logger.php @@ -33,7 +33,8 @@ public static function log() { return static::me()->getLogger(); } - public static function error($message, $context) + + public static function error($message, $context = []) { if ($context instanceof \Throwable) { $context = [ @@ -47,10 +48,27 @@ public static function error($message, $context) } } - return static::log()->err($message, $context); + return static::log()->error($message, $context); } + public static function dbg($message, $context = []) + { + if ($context instanceof \Throwable) { + $context = [ + 'exception' => get_class($context), + 'trace' =>$context->getTraceAsString() + ]; + if ($context instanceof Iface\ExceptionWithParams) { + $context = [ + 'params' => $context->getParams() + ]; + } + } + + return static::log()->debug($message, $context); + } + public function setLogger(\Psr\Log\LoggerInterface $logger) { $this->logger = $logger; @@ -60,30 +78,64 @@ public function setLogger(\Psr\Log\LoggerInterface $logger) private function getDefaultCliFormater() { - return new Monolog\Formatter\LineFormatter("%message%"); + return new Monolog\Formatter\LineFormatter("%message%\n"); + } + + private function getDefaultSingleLineFormater() + { + return new Monolog\Formatter\LineFormatter('%message%'); } private function getLoggerDefault() { - $logger = new Monolog\Logger('Language'); - $logger - ->pushHandler( - new Monolog\Handler\StreamHandler( - 'php://stderr', - Monolog\Logger::WARNING + $logger = (new Monolog\Logger('Language')); + + + if (!getenv('FINXLOG_DEBUG')) { + $logger + ->pushHandler( + new Monolog\Handler\StreamHandler( + 'php://stderr', + Monolog\Logger::WARNING + ) + ); + } elseif (getenv('FINXLOG_DEBUG') <= Monolog\Logger::DEBUG) { + $logger + ->pushHandler( + ( + new Monolog\Handler\StreamHandler( + 'php://stderr', + Monolog\Logger::INFO + ) + ) + ->setFormatter( + new Monolog\Formatter\LineFormatter("\n") + ) ) - ) - ->pushHandler( + ->pushHandler( + ( + new Monolog\Handler\StreamHandler( + 'php://stderr', + Monolog\Logger::DEBUG + ) + ) + ->setFormatter( + $this->getDefaultSingleLineFormater() + ) + ); + } else { + $logger->pushHandler( ( new Monolog\Handler\StreamHandler( 'php://stdout', - getenv('FINXLOG_DEBUG') ? Monolog\Logger::DEBUG : Monolog\Logger::INFO + getenv('FINXLOG_DEBUG') > Monolog\Logger::DEBUG + ? getenv('FINXLOG_DEBUG') + : Monolog\Logger::INFO ) ) - ->setFormatter( - $this->getDefaultCliFormater() - ) + ->setFormatter($this->getDefaultCliFormater()) ); + } return $logger; } diff --git a/src/Classes/Module/Ratchet/QuotationServer.php b/src/Classes/Module/Ratchet/QuotationServer.php index b6c1e6c..4d1ec67 100644 --- a/src/Classes/Module/Ratchet/QuotationServer.php +++ b/src/Classes/Module/Ratchet/QuotationServer.php @@ -1,6 +1,7 @@ io_server) { - assert(getenv('FINXLOG_WEBSOCKET_PORT') > 0); + assert(getenv('FINXLOG_WEBSOCKET_LISTEN_PORT') > 0); $this->io_server = IoServer::factory( new HttpServer( @@ -23,7 +24,8 @@ public function getIoServer() new QuotationWebSocketDelivery() ) ), - getenv('FINXLOG_WEBSOCKET_PORT') + getenv('FINXLOG_WEBSOCKET_LISTEN_PORT'), + getenv('FINXLOG_WEBSOCKET_LISTEN_INTERFACE') ?: '0.0.0.0' ); } @@ -39,6 +41,7 @@ public function setIoServer(IoServer $io_server) public function run() { + Logger::dbg(':ws_wait:'); $this->getIoServer()->run(); } } diff --git a/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php b/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php index 0a44187..9a7b56c 100644 --- a/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php +++ b/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php @@ -15,30 +15,34 @@ class QuotationWebSocketDelivery implements \Ratchet\MessageComponentInterface use WithQueueConnector; const QUOTATION_ALL = SearchQuotation::QUOTATION_ALL; + /** * @var \SplObjectStorage|ConnectionInterface[] */ - protected $clients = []; + protected $clients; /** - * @var ConnectionInterface[][][] + * @var Subscribers */ - protected $subscribers = []; + protected $subscribers; /** * @var bool[] */ protected $service_by_resource; - public function __construct() + public function getClients() { - $this->clients = new \SplObjectStorage; + if (!$this->clients) { + $this->clients = new \SplObjectStorage; + } + return $this->clients; } /** * is service => allow incoming - * @param ConnectionInterface $conn + * @param ConnectionInterface|RFC6455\Connection $conn * @return bool */ protected function isService(ConnectionInterface $conn, $msg = null) @@ -49,15 +53,11 @@ protected function isService(ConnectionInterface $conn, $msg = null) ) { return $this->service_by_resource[$conn->resourceId]; } - assert(strlen( - getenv('FINXLOG_WEBSOCKET_SERVICE_PATH') - . getenv('FINXLOG_WEBSOCKET_SERVICE_FILTER_ADDR_REGEXP') - )); $is_service = true; if (strlen(getenv('FINXLOG_WEBSOCKET_SERVICE_FILTER_ADDR_REGEXP'))) { - $is_service = (bool)preg_match( + $is_service = $is_service && (bool) preg_match( '~' . getenv('FINXLOG_WEBSOCKET_SERVICE_FILTER_ADDR_REGEXP') . '~iu', $conn->remoteAddress ); @@ -69,7 +69,7 @@ protected function isService(ConnectionInterface $conn, $msg = null) $is_service && $conn instanceof RFC6455\Connection && $conn->WebSocket->request - && strstr( + && false !== strstr( $conn->WebSocket->request->getPath(), getenv('FINXLOG_WEBSOCKET_SERVICE_PATH'), 1 @@ -81,7 +81,6 @@ protected function isService(ConnectionInterface $conn, $msg = null) $e->getMessage(), ['e' => $e] ); - $is_service = false; } @@ -90,27 +89,24 @@ protected function isService(ConnectionInterface $conn, $msg = null) public function onOpen(ConnectionInterface $conn) { - if (!$this->isService($conn)) { - //waiting for manual subscribe - //$this->clients['quotation'][static::QUOTATION_ALL]->attach($conn); - } - - Logger::log()->debug( - "WS: New " - . ( + if (getenv('FINXLOG_DEBUG')) { + Logger::log()->info( + "WS: New " + . ( $this->isService($conn) ? 'service' : 'client' - ) - . " connection: {$conn->resourceId}", - [ - 'is_service' => $this->isService($conn), - 'conn' => $conn - ] - ); + ) + . " connection: {$conn->resourceId}", + [ + 'is_service' => $this->isService($conn), + 'conn' => $conn + ] + ); + } } - public function getPreparedMessage(ConnectionInterface $conn, $msg) + public function getPreparedMessage($msg, ConnectionInterface $conn = null) { $message = json_decode($msg, true); if ( @@ -118,7 +114,7 @@ public function getPreparedMessage(ConnectionInterface $conn, $msg) || !is_array($message) || empty($message['type']) ) { - throw new WrongParams("WS: !msg: $msg"); + throw new WrongParams("WS: !msg[type]: $msg"); } if ( !empty($message['quotation']) @@ -144,16 +140,18 @@ public function getPreparedMessage(ConnectionInterface $conn, $msg) public function onMessage(ConnectionInterface $conn, $msg) { - Logger::log()->debug( - "WS msg from #{$conn->resourceId}: {$msg}", - [ - 'conn' => $conn, - 'is_service' => $this->isService($conn), - 'conn_count' => count($this->service_by_resource), - ] - ); + if (getenv('FINXLOG_DEBUG')) { + Logger::log()->info( + "WS " . ($this->isService($conn) ? 'service' : 'client'). " msg from #{$conn->resourceId}:" . mb_substr($msg,0,100) . '...', + [ + 'conn' => $conn, + 'is_service' => $this->isService($conn), + 'conn_count' => count($this->service_by_resource), + ] + ); + } - if (!$this->isService($conn) && strlen($msg) > 1000) { + if (!$this->isService($conn) && strlen($msg) > 100000) { Logger::log()->warning( "WS: msg with " . round(strlen($msg) / 1000). "kb", [ @@ -165,7 +163,7 @@ public function onMessage(ConnectionInterface $conn, $msg) } try { - $message = $this->getPreparedMessage($conn, $msg); + $message = $this->getPreparedMessage($msg, $conn); if ($this->isService($conn, $message)) { $this->addServiceIncoming($conn, $message); } @@ -208,7 +206,7 @@ protected function addServiceIncoming(ConnectionInterface $conn, array $message) ])); break; case 'all': - foreach ($this->clients as $client) { + foreach ($this->getClients() as $client) { if (!$this->isService($client)) { $conn->send(json_encode($message)); } @@ -216,17 +214,29 @@ protected function addServiceIncoming(ConnectionInterface $conn, array $message) break; case 'send': foreach ( - $this->subscribers - [$message['quotation']] - [empty($message['agg_period']) ? 0 : $message['agg_period']] - as $subscriber + $this->getSubscribers()->get($message) + as $subscribe ) { - $subscriber['ws']->send(json_encode($message)); + $subscribe['ws']->send(json_encode($message)); } + Logger::log()->debug(':2js:'); + break; } } + /** + * @return Subscribers + */ + public function getSubscribers() + { + if (!$this->subscribers) { + $this->subscribers = new Subscribers; + } + + return $this->subscribers; + } + /** * set client opts * @param ConnectionInterface $conn @@ -234,73 +244,72 @@ protected function addServiceIncoming(ConnectionInterface $conn, array $message) */ protected function addClientIncoming(ConnectionInterface $conn, array $message) { - assert(!empty($message['type'])); - if ($message['type'] != 'subscribe') { - return false; + if (empty($message['type'])) { + Logger::log()->warning('WS client? message without type'); + return $this; } - $agg_period = 0; - if (!empty($message['doji'])) { - $all_period = (new QuotationAgg)->getAggPeriod()[strtoupper($message['doji'])]; - if (empty($all_period[strtoupper($message['doji'])])) { - throw (new WrongParams('WS: wrong doji period on subscribe')) - ->setParams(['doji']); - } - $agg_period = $all_period[strtoupper($message['doji'])]; + + switch ($message['type']) { + case 'subscribe': + $agg_period = 0; + if (!empty($message['doji'])) { + $all_period = (new QuotationAgg)->getAggPeriod()[strtoupper($message['doji'])]; + if (empty($all_period[strtoupper($message['doji'])])) { + throw (new WrongParams('WS: wrong doji period on subscribe')) + ->setParams(['doji']); + } + $message['agg_period'] = $all_period[strtoupper($message['doji'])]; + } + + /** + * [USDEUR][3600][#123] = [ ... ]; + */ + $this->getSubscribers()->add( + ['ws' => $conn] + $message + ['agg' => null,] + ); + + //load previous period + $this->addJob($new_job = [ + 'quotation' => $message['quotation'], + 'agg' => !empty($message['doji']) ? 'doji' : null, + 'agg_period' => !empty($message['doji']) ? $message['doji'] : null, + ]); + Logger::log()->info('AMQP add:' . json_encode($new_job)); + break; + case 'unsubscribe': + $this->getSubscribers()->drop($conn, $message); + break; } - /** - * [USDEUR][3600][#123] = [ ... ]; - */ - $this->subscribers - [$message['quotation']] - [$agg_period] - [$conn->resourceId] - = [ - 'ws' => $conn, - 'agg' => !empty($message['doji']) ? 'doji' : null, - ]; - - //load previous period - $this->addJob([ - 'quotation' => $message['quotation'], - 'agg' => !empty($message['doji']) ? 'doji' : null, - 'agg_period' => !empty($message['doji']) ? $message['doji'] : null, - ]); - - //@todo add queue + return $this; } public function onClose(ConnectionInterface $conn) { - Logger::log()->debug( - "WS:Disconnect with #{$conn->resourceId}", - [ - 'conn' => $conn, - ] - ); - - unset($this->service_by_resource[$conn->resourceId]); - - foreach ($this->subscribers as $k1 => $v1) { - foreach ($v1 as $k2=>$v2) { - unset($this->subscribers[$k1][$k2][$conn->resourceId]); - } - $this->subscribers[$k1] = array_filter($this->subscribers[$k1]); + if (getenv('FINXLOG_DEBUG')) { + Logger::log()->info( + "WS:Disconnect with #{$conn->resourceId}", + [ + 'conn' => $conn, + ] + ); } - $this->subscribers = array_filter($this->subscribers); - $this->clients->detach($conn); + unset($this->service_by_resource[$conn->resourceId]); + $this->getSubscribers()->drop($conn); + $this->getClients()->detach($conn); } public function onError(ConnectionInterface $conn, \Exception $e) { - Logger::log()->debug( - "WS: " .get_class($e) . ": {$e->getMessage()} with #{$conn->resourceId}", - [ - //'this', $this, - 'e' => $e, - 'conn' => $conn - ] - ); + if (getenv('FINXLOG_DEBUG')) { + Logger::log()->info( + "WS: " . get_class($e) . ": {$e->getMessage()} with #{$conn->resourceId}", + [ + 'e' => $e, + 'conn' => $conn + ] + ); + } $conn->close(); } diff --git a/src/Classes/Module/Ratchet/Subscribers.php b/src/Classes/Module/Ratchet/Subscribers.php new file mode 100644 index 0000000..960a7e8 --- /dev/null +++ b/src/Classes/Module/Ratchet/Subscribers.php @@ -0,0 +1,106 @@ +getKeyByMessage($key); + } + + return empty($this->subscribers[$key]) + ? [] + : $this->subscribers[$key]; + } + + + /** + * @param RFC6455\Connection|ConnectionInterface $conn + * @return array + */ + public function getByConn(ConnectionInterface $conn) + { + if (empty($this->map_by_conn[$conn->resourceId])) { + return []; + } + $result = []; + foreach ($this->map_by_conn[$conn->resourceId] as $key) { + if (empty($this->subscribers[$key][$conn->resourceId])) { + Logger::error('subscribers: mapped key is missing'); + } else { + $result[] = $this->subscribers[$key][$conn->resourceId]; + } + } + + return $result; + } + + /** + * @param RFC6455\Connection|ConnectionInterface $conn + * @param string $key + * @return $this + */ + public function drop(ConnectionInterface $conn, $key = null) + { + if ($key) { + assert(isset($this->map_by_conn[$conn->resourceId][$key])); + assert(isset($this->subscribers[$key][$conn->resourceId])); + unset($this->map_by_conn[$conn->resourceId][$key]); + unset($this->subscribers[$key][$conn->resourceId]); + + if (empty($this->subscribers[$key])) { + unset($this->subscribers[$key]); + } + } elseif (isset($this->map_by_conn[$conn->resourceId])) { + foreach ((array) $this->map_by_conn[$conn->resourceId] as $key) { + unset($this->subscribers[$key][$conn->resourceId]); + if (empty($this->subscribers[$key])) { + unset($this->subscribers[$key]); + } + } + unset($this->map_by_conn[$conn->resourceId]); + } + + return $this; + } + + /** + * @param string|array $message + * @return $this + */ + public function add(array $message) + { + $key = $this->getKeyByMessage($message); + $this->subscribers[$key][$message['ws']->resourceId] = $message; + $this->map_by_conn[$message['ws']->resourceId] = $key; + + return $this; + } +} diff --git a/www/index.html b/www/index.html index f51edf6..626791a 100644 --- a/www/index.html +++ b/www/index.html @@ -3,87 +3,9 @@ +
- diff --git a/www/js/.gitkeep b/www/js/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/www/js/app.js b/www/js/app.js new file mode 100644 index 0000000..71779ce --- /dev/null +++ b/www/js/app.js @@ -0,0 +1,140 @@ +Highcharts.setOptions({ + global: { + useUTC: false + } +}); + +var FinXLog = { + config: {}, + start: function() { + this.ws.setup(); + for (var i = 0; i < this.graph.length; ++i) { + this.graph[i].setup(); + } + this.ws.subscribe('_ALL'); + }, + ws: { + subscribes: [ + {type: 'subscribe', quotation: '_ALL', doji: null} + ], + conn: null, + setup: function () { + this.conn = new WebSocket(this.getWsUrl()); + this.conn.onopen = function() { + if (FinXLog.config.FINXLOG_DEBUG) { + console.log('WS: open'); + } + for (var i = 0; i < FinXLog.ws.subscribes.length; ++i) { + FinXLog.ws.send(FinXLog.ws.subscribes[i]); + } + }; + + this.conn.onmessage = function (msg) { + console.log('incoming: '); + console.log(msg.data); + }; + }, + subscribe: function (subj, doji, stop) { + var request = { + 'type': stop ? 'unsubscribe' : 'subscribe', + 'quotation': subj, + 'doji': doji ? doji : null + }; + this.send(request); + if (stop) { + for (var i = 0; i < FinXLog.ws.subscribes.length; ++i) { + if (JSON.stringify(FinXLog.ws.subscribes[i]) == JSON.stringify(request)) { + //@todo + //FinXLog.ws.subscribes.remove + console.log('remove:'); + console.log(FinXLog.ws.subscribes[i]); + } + + } + } else { + FinXLog.ws.subscribes.push(request) + } + }, + send: function (obj) { + console.log(obj); + this.conn.send(JSON.stringify(obj)); + }, + getWsUrl: function () + { + return 'ws://' + + (FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_HOST ? FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_HOST : FinXLog.config.FINXLOG_DOMAIN) + + (FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_PORT ? ':' + FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_PORT : '') + + FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_PATH ; + } + }, + graph: [ + { + conn: null, + setup: function () { + this.conn = new Highcharts.Chart(this.graph_opts); + }, + graph_opts: { + title: {text: 'Real Time Samples'}, + xAxis: { + type: 'datetime', + tickPixelInterval: 100 + }, + yAxis: { + title: {text: 'Samples'}, + tickInterval: 10, + min: 0, + max: 100 + }, + tooltip: { + formatter: function () { + return '' + this.series.name + '
' + + JSON.stringify([ + Highcharts.dateFormat('%Y-%m-%d %H:%M:%S', this.x), + this.y + ]); + } + }, + chart: { + type: 'spline', + renderTo: 'graph_container', + events: { + load: function () { + if (FinXLog.config.FINXLOG_DEBUG) { + console.log('chart load'); + } + } + } + }, + series: [{ + name: 'tst', + data: (function () { + var data = [], + time = (new Date()).getTime(), + i; + for (i = -19; i <= 0; i++) { + data.push({ + x: time + (i * 1000), + y: 0 + }); + } + return data; + })() + }], + draw: function (data) { + console.log('draw:'); + console.log(data); + } + } + } + ] +}; + +includeJS('js/config_autobuild.js'); + +function includeJS(url) +{ + var script = document.createElement('script'); + script.src = url; + script.type = 'text/javascript'; + document.getElementsByTagName('head')[0].appendChild(script); +} \ No newline at end of file From a378db3eac108b025a778c2a124ca9c439d5318c Mon Sep 17 00:00:00 2001 From: bag Date: Sat, 18 Jun 2016 05:46:06 +0300 Subject: [PATCH 10/23] add manual dev/test tool --- command/test/add_ws_msg_client.php | 28 +++++++++ command/test/add_ws_msg_service.php | 62 ++++++++++++++++++- .../Module/Connector/RatchetClient.php | 7 ++- 3 files changed, 94 insertions(+), 3 deletions(-) create mode 100755 command/test/add_ws_msg_client.php diff --git a/command/test/add_ws_msg_client.php b/command/test/add_ws_msg_client.php new file mode 100755 index 0000000..369cad4 --- /dev/null +++ b/command/test/add_ws_msg_client.php @@ -0,0 +1,28 @@ +#!/usr/bin/php +then(function(\Ratchet\Client\WebSocket $conn) use ($listen_count) { + $conn->on('message', function($msg) use ($conn, $listen_count) { + echo "Received: {$msg}\n"; + if ($listen_count) { + static $count = 0; + if (++$count >= $listen_count) { + $conn->close(); + } + } + + }); + + $conn->send('{"type":"subscribe","quotation":"_ALL"}'); + $conn->send('{"type":"subscribe","quotation":"BTCUSD"}'); +}); \ No newline at end of file diff --git a/command/test/add_ws_msg_service.php b/command/test/add_ws_msg_service.php index 295de1a..c3f18f7 100755 --- a/command/test/add_ws_msg_service.php +++ b/command/test/add_ws_msg_service.php @@ -9,5 +9,63 @@ $client->addWsMessage(['type' => 'test']); $client->addWsMessage(' - {"type":"send","quotation":"_ALL","agg_period":null,"quotations":[{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/09 04:05:51","B":2.151},{"S":"TST","T":"2016\/06\/02 04:05:51","B":4},{"S":"BTCUSD","T":"2016\/06\/06 01:05:51","B":1.5},{"S":"BTCUSD","T":"2016\/06\/11 11:00:00","B":99},{"S":"BTCUSD","T":"2016\/06\/11 00:00:01","B":1},{"S":"BTCUSD","T":"2016\/06\/11 03:53:42","B":567.5},{"S":"TST","T":"2016\/06\/01 04:05:51","B":3.1},{"S":"BTCUSD","T":"2016\/06\/06 01:00:51","B":2},{"S":"BTCUSD","T":"2017\/06\/11 12:00:00","B":1},{"S":"BTCUSD","T":"2017\/06\/11 11:00:00","B":99},{"S":"BTCUSD","T":"2016\/06\/11 12:00:01","B":1},{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/10 04:05:51","B":1.151},{"S":"BTCUSD","T":"2015\/06\/07 04:05:51","B":2},{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/07 03:05:51","B":3.5},{"S":"TST2","T":"2016\/06\/06 04:05:51","B":5},{"S":"TST2","T":"2016\/06\/07 04:05:51","B":6},{"S":"BTCUSD","T":"2016\/06\/11 23:00:00","B":99},{"S":"BTCUSD","T":"2016\/06\/11 23:00:00","B":999},{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151},{"S":"BTCUSD","T":"2016\/06\/08 04:05:51","B":3.1},{"S":"BTCUSD","T":"2017\/06\/07 04:05:51","B":1},{"S":"BTCUSD","T":"2016\/06\/11 12:00:00","B":1},{"S":"BTCUSD","T":"2016\/06\/11 00:00:01","B":10}]} - '); \ No newline at end of file + { + "type":"send", + "quotation":"_ALL", + "agg_period":null, + "quotations":[ + {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016\/06\/09 04:05:51","B":2.151}, + {"S":"TST","T":"2016\/06\/02 04:05:51","B":4}, + {"S":"BTCUSD","T":"2016\/06\/06 01:05:51","B":1.5}, + {"S":"BTCUSD","T":"2016\/06\/11 11:00:00","B":99}, + {"S":"BTCUSD","T":"2016\/06\/11 00:00:01","B":1}, + {"S":"BTCUSD","T":"2016\/06\/11 03:53:42","B":567.5}, + {"S":"TST","T":"2016\/06\/01 04:05:51","B":3.1}, + {"S":"BTCUSD","T":"2016\/06\/06 01:00:51","B":2}, + {"S":"BTCUSD","T":"2017\/06\/11 12:00:00","B":1}, + {"S":"BTCUSD","T":"2017\/06\/11 11:00:00","B":99}, + {"S":"BTCUSD","T":"2016\/06\/11 12:00:01","B":1}, + {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016\/06\/10 04:05:51","B":1.151}, + {"S":"BTCUSD","T":"2015\/06\/07 04:05:51","B":2}, + {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016\/06\/07 03:05:51","B":3.5}, + {"S":"TST2","T":"2016\/06\/06 04:05:51","B":5}, + {"S":"TST2","T":"2016\/06\/07 04:05:51","B":6}, + {"S":"BTCUSD","T":"2016\/06\/11 23:00:00","B":99}, + {"S":"BTCUSD","T":"2016\/06\/11 23:00:00","B":999}, + {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016\/06\/08 04:05:51","B":3.1}, + {"S":"BTCUSD","T":"2017\/06\/07 04:05:51","B":1}, + {"S":"BTCUSD","T":"2016\/06\/11 12:00:00","B":1}, + {"S":"BTCUSD","T":"2016\/06\/11 00:00:01","B":10} + ] + } +'); + +//slice of real +$client->addWsMessage(' + { + "type":"send", + "quotation":"BTCUSD", + "agg_period":null, + "quotations":[ + {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016\/06\/09 04:05:51","B":2.151}, + {"S":"BTCUSD","T":"2016\/06\/06 01:05:51","B":1.5}, + {"S":"BTCUSD","T":"2016\/06\/11 11:00:00","B":99}, + {"S":"BTCUSD","T":"2016\/06\/11 03:53:42","B":567.5}, + {"S":"BTCUSD","T":"2016\/06\/06 01:00:51","B":2}, + {"S":"BTCUSD","T":"2017\/06\/11 12:00:00","B":1}, + {"S":"BTCUSD","T":"2017\/06\/11 11:00:00","B":99}, + {"S":"BTCUSD","T":"2016\/06\/11 12:00:01","B":1}, + {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151} + ] + } +'); \ No newline at end of file diff --git a/src/Classes/Module/Connector/RatchetClient.php b/src/Classes/Module/Connector/RatchetClient.php index d2154a6..bef603a 100644 --- a/src/Classes/Module/Connector/RatchetClient.php +++ b/src/Classes/Module/Connector/RatchetClient.php @@ -100,7 +100,12 @@ function(\Ratchet\Client\WebSocket $conn) use ($message, $event_loop, $incomingCallback) { if (!is_string($message)) { - $message = json_encode($message); + $message = json_encode( + $message, + getenv('FINXLOG_DEBUG') + ? JSON_PRETTY_PRINT + : null + ); } if (getenv('FINXLOG_DEBUG')) { Logger::log()->info("AMQP2WS(service) send try:\t{$message}"); From 07f2dddfe12760e9024cdc5cae5e7587db121cd8 Mon Sep 17 00:00:00 2001 From: bag Date: Sat, 18 Jun 2016 07:23:09 +0300 Subject: [PATCH 11/23] fix ws loop reconnect. work only on second request --- command/test/add_ws_msg_client.php | 3 +- command/test/add_ws_msg_service.php | 54 ++++--------------- .../Module/Connector/RatchetClient.php | 30 ++++++++--- .../Ratchet/QuotationWebSocketDelivery.php | 36 ++++++++++--- www/index.html | 1 + www/js/app.js | 44 ++++++++++++--- 6 files changed, 101 insertions(+), 67 deletions(-) diff --git a/command/test/add_ws_msg_client.php b/command/test/add_ws_msg_client.php index 369cad4..037181e 100755 --- a/command/test/add_ws_msg_client.php +++ b/command/test/add_ws_msg_client.php @@ -13,7 +13,7 @@ \Ratchet\Client\connect('ws://127.0.0.1:8080/ws')->then(function(\Ratchet\Client\WebSocket $conn) use ($listen_count) { $conn->on('message', function($msg) use ($conn, $listen_count) { - echo "Received: {$msg}\n"; + echo "\nReceived: " . substr($msg, 0, 100). "...\n\n"; if ($listen_count) { static $count = 0; if (++$count >= $listen_count) { @@ -23,6 +23,7 @@ }); + $conn->send('{"type": "quotations" }'); $conn->send('{"type":"subscribe","quotation":"_ALL"}'); $conn->send('{"type":"subscribe","quotation":"BTCUSD"}'); }); \ No newline at end of file diff --git a/command/test/add_ws_msg_service.php b/command/test/add_ws_msg_service.php index c3f18f7..d231e0a 100755 --- a/command/test/add_ws_msg_service.php +++ b/command/test/add_ws_msg_service.php @@ -3,11 +3,7 @@ require_once __DIR__ . '/../../vendor/autoload.php'; $client = (new \FinXLog\Module\ClientQueue\SearchQuotation); -$client->addWsMessage(['type' => 'test']); -$client->addWsMessage(['type' => 'test']); -$client->addWsMessage(['type' => 'test']); -$client->addWsMessage(['type' => 'test']); - +//slice of real : all $client->addWsMessage(' { "type":"send", @@ -15,39 +11,12 @@ "agg_period":null, "quotations":[ {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, - {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, - {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, - {"S":"BTCUSD","T":"2016\/06\/09 04:05:51","B":2.151}, - {"S":"TST","T":"2016\/06\/02 04:05:51","B":4}, - {"S":"BTCUSD","T":"2016\/06\/06 01:05:51","B":1.5}, - {"S":"BTCUSD","T":"2016\/06\/11 11:00:00","B":99}, - {"S":"BTCUSD","T":"2016\/06\/11 00:00:01","B":1}, - {"S":"BTCUSD","T":"2016\/06\/11 03:53:42","B":567.5}, - {"S":"TST","T":"2016\/06\/01 04:05:51","B":3.1}, - {"S":"BTCUSD","T":"2016\/06\/06 01:00:51","B":2}, - {"S":"BTCUSD","T":"2017\/06\/11 12:00:00","B":1}, - {"S":"BTCUSD","T":"2017\/06\/11 11:00:00","B":99}, - {"S":"BTCUSD","T":"2016\/06\/11 12:00:01","B":1}, - {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, - {"S":"BTCUSD","T":"2016\/06\/10 04:05:51","B":1.151}, - {"S":"BTCUSD","T":"2015\/06\/07 04:05:51","B":2}, - {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, - {"S":"BTCUSD","T":"2016\/06\/07 03:05:51","B":3.5}, - {"S":"TST2","T":"2016\/06\/06 04:05:51","B":5}, - {"S":"TST2","T":"2016\/06\/07 04:05:51","B":6}, - {"S":"BTCUSD","T":"2016\/06\/11 23:00:00","B":99}, - {"S":"BTCUSD","T":"2016\/06\/11 23:00:00","B":999}, - {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, - {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, - {"S":"BTCUSD","T":"2016\/06\/08 04:05:51","B":3.1}, - {"S":"BTCUSD","T":"2017\/06\/07 04:05:51","B":1}, - {"S":"BTCUSD","T":"2016\/06\/11 12:00:00","B":1}, - {"S":"BTCUSD","T":"2016\/06\/11 00:00:01","B":10} + {"S":"TST","T":"2016\/06\/02 04:05:51","B":4} ] } '); -//slice of real +//slice of real: BTCUSD $client->addWsMessage(' { "type":"send", @@ -55,17 +24,12 @@ "agg_period":null, "quotations":[ {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, - {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, - {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, - {"S":"BTCUSD","T":"2016\/06\/09 04:05:51","B":2.151}, - {"S":"BTCUSD","T":"2016\/06\/06 01:05:51","B":1.5}, - {"S":"BTCUSD","T":"2016\/06\/11 11:00:00","B":99}, - {"S":"BTCUSD","T":"2016\/06\/11 03:53:42","B":567.5}, - {"S":"BTCUSD","T":"2016\/06\/06 01:00:51","B":2}, - {"S":"BTCUSD","T":"2017\/06\/11 12:00:00","B":1}, - {"S":"BTCUSD","T":"2017\/06\/11 11:00:00","B":99}, - {"S":"BTCUSD","T":"2016\/06\/11 12:00:01","B":1}, {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151} ] } -'); \ No newline at end of file +'); + + +$client->addWsMessage(['type' => 'conn_count']); +$client->addWsMessage(['type' => 'all', 'xx' => 123]); +$client->addWsMessage(['type' => 'test']); diff --git a/src/Classes/Module/Connector/RatchetClient.php b/src/Classes/Module/Connector/RatchetClient.php index bef603a..5227210 100644 --- a/src/Classes/Module/Connector/RatchetClient.php +++ b/src/Classes/Module/Connector/RatchetClient.php @@ -86,19 +86,32 @@ public function close() return $this; } + public function reconnect() + { + $this->promise = null; + $this->event_loop = null; + $this->connector = null; + } public function send($message, callable $incomingCallback = null) { + //reconnec //force new connect - //$this->promise = null; - //$this->event_loop = null; - //$this->connector = null; + + $self = $this; + $is_first = empty($this->event_loop); $event_loop = $this->getEventLoop(); - $this->getPromise()->then( function(\Ratchet\Client\WebSocket $conn) - use ($message, $event_loop, $incomingCallback) + use ($message, $event_loop, $incomingCallback, $self) { + + $conn->on('error', function($error) use ($conn, $self, $message) { + //it's check only on second request + $self->reconnect(); + $self->send($message);//without - 3 iterations + }); + if (!is_string($message)) { $message = json_encode( $message, @@ -128,7 +141,7 @@ function ($code = null, $reason = null) } }); //@todo check that the socket stream is empty(released) without on:event - $conn->on('messa;ge', function ($message) use ($conn) { + $conn->on('message', function ($message) use ($conn) { Logger::log()->warning( "AMQP2WS(service) ignore message: {$message}" ); @@ -136,9 +149,11 @@ function ($code = null, $reason = null) } } $conn->send($message); - //exit from EventLoop $event_loop->stop(); + + + }, function (\Throwable $e) use ($event_loop, $message, $incomingCallback) @@ -156,6 +171,7 @@ function (\Throwable $e) if ($is_first) { $event_loop->run(); } + //push all queue $event_loop->tick(); diff --git a/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php b/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php index 9a7b56c..8299d93 100644 --- a/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php +++ b/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php @@ -87,6 +87,9 @@ protected function isService(ConnectionInterface $conn, $msg = null) return $this->service_by_resource[$conn->resourceId] = $is_service; } + /** + * @param ConnectionInterface|RFC6455\Connection $conn + */ public function onOpen(ConnectionInterface $conn) { if (getenv('FINXLOG_DEBUG')) { @@ -138,11 +141,15 @@ public function getPreparedMessage($msg, ConnectionInterface $conn = null) return $message; } + /** + * @param ConnectionInterface|RFC6455\Connection $conn + */ public function onMessage(ConnectionInterface $conn, $msg) { if (getenv('FINXLOG_DEBUG')) { Logger::log()->info( - "WS " . ($this->isService($conn) ? 'service' : 'client'). " msg from #{$conn->resourceId}:" . mb_substr($msg,0,100) . '...', + "WS " . ($this->isService($conn) ? 'service' : 'client'). + " msg from #{$conn->resourceId}:" . mb_substr($msg, 0, 100) . '...', [ 'conn' => $conn, 'is_service' => $this->isService($conn), @@ -187,11 +194,10 @@ public function onMessage(ConnectionInterface $conn, $msg) ); } } - /** * delivery to clients - * @param ConnectionInterface $conn - * @param string $msg + * @param ConnectionInterface|RFC6455\Connection $conn + * @param array $message */ protected function addServiceIncoming(ConnectionInterface $conn, array $message) { @@ -217,7 +223,7 @@ protected function addServiceIncoming(ConnectionInterface $conn, array $message) $this->getSubscribers()->get($message) as $subscribe ) { - $subscribe['ws']->send(json_encode($message)); + $subscribe['ws']->send(json_encode($message, JSON_HEX_TAG)); } Logger::log()->debug(':2js:'); @@ -239,8 +245,8 @@ public function getSubscribers() /** * set client opts - * @param ConnectionInterface $conn - * @param string $msg + * @param ConnectionInterface|RFC6455\Connection $conn + * @param array $message */ protected function addClientIncoming(ConnectionInterface $conn, array $message) { @@ -250,8 +256,14 @@ protected function addClientIncoming(ConnectionInterface $conn, array $message) } switch ($message['type']) { + case 'quotations': + //mock + $conn->send(json_encode([ + "type" => 'quotations', + 'quotations' => ['BTCUSD','USDBTC','USDEUR','EURUSD'] + ])); + break; case 'subscribe': - $agg_period = 0; if (!empty($message['doji'])) { $all_period = (new QuotationAgg)->getAggPeriod()[strtoupper($message['doji'])]; if (empty($all_period[strtoupper($message['doji'])])) { @@ -278,11 +290,15 @@ protected function addClientIncoming(ConnectionInterface $conn, array $message) break; case 'unsubscribe': $this->getSubscribers()->drop($conn, $message); + Logger::log()->info('AMQP unsubscribe'); break; } return $this; } + /** + * @param ConnectionInterface|RFC6455\Connection $conn + */ public function onClose(ConnectionInterface $conn) { if (getenv('FINXLOG_DEBUG')) { @@ -299,6 +315,10 @@ public function onClose(ConnectionInterface $conn) $this->getClients()->detach($conn); } + /** + * @param ConnectionInterface|RFC6455\Connection $conn + * @param \Exception $e + */ public function onError(ConnectionInterface $conn, \Exception $e) { if (getenv('FINXLOG_DEBUG')) { diff --git a/www/index.html b/www/index.html index 626791a..ba0450f 100644 --- a/www/index.html +++ b/www/index.html @@ -6,6 +6,7 @@ +
diff --git a/www/js/app.js b/www/js/app.js index 71779ce..7f4450b 100644 --- a/www/js/app.js +++ b/www/js/app.js @@ -13,26 +13,58 @@ var FinXLog = { } this.ws.subscribe('_ALL'); }, + dbg: function(msg, obj) { + if (!FinXLog.config.FINXLOG_DEBUG) return; + console.log(msg); + if (obj) console.log(obj); + }, ws: { subscribes: [ {type: 'subscribe', quotation: '_ALL', doji: null} ], + query_onstart: [//just for non-static + {type: 'quotations'} + ], conn: null, setup: function () { this.conn = new WebSocket(this.getWsUrl()); this.conn.onopen = function() { if (FinXLog.config.FINXLOG_DEBUG) { - console.log('WS: open'); + FinXLog.dbg('WS: open'); + } + for (var i = 0; i < FinXLog.ws.query_onstart.length; ++i) { + FinXLog.ws.conn.send(FinXLog.ws.query_onstart[i]); } + for (var i = 0; i < FinXLog.ws.subscribes.length; ++i) { FinXLog.ws.send(FinXLog.ws.subscribes[i]); } }; - this.conn.onmessage = function (msg) { - console.log('incoming: '); - console.log(msg.data); - }; + this.conn.onmessage = this.onmessage; + }, + onmessage: function (request) { + var message; + try { + message = JSON.parse(request); + FinXLog.dbg('incoming:'); + FinXLog.dbg(message.data.type); + } catch (err) { + FinXLog.dbg('wrong incoming:'); + FinXLog.dbg(err); + FinXLog.dbg(request.data); + throw err + } + + + switch (message.data.type) { + case 'quotations': { + for (var i = 0; i < msg.data.quotations.length; ++i) { + this.dbg(msg.data.quotations[i]); + } + break; + } + } }, subscribe: function (subj, doji, stop) { var request = { @@ -56,7 +88,7 @@ var FinXLog = { } }, send: function (obj) { - console.log(obj); + FinXLog.dbg('send:', obj); this.conn.send(JSON.stringify(obj)); }, getWsUrl: function () From 0db5586b14c8717311f5ffbb919b9f54c0ed2ce5 Mon Sep 17 00:00:00 2001 From: bag Date: Sat, 18 Jun 2016 07:27:48 +0300 Subject: [PATCH 12/23] fix ws loop reconnect. work only on second request --- src/Classes/Module/Connector/RatchetClient.php | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Classes/Module/Connector/RatchetClient.php b/src/Classes/Module/Connector/RatchetClient.php index 5227210..2c3c502 100644 --- a/src/Classes/Module/Connector/RatchetClient.php +++ b/src/Classes/Module/Connector/RatchetClient.php @@ -86,12 +86,6 @@ public function close() return $this; } - public function reconnect() - { - $this->promise = null; - $this->event_loop = null; - $this->connector = null; - } public function send($message, callable $incomingCallback = null) { //reconnec @@ -106,10 +100,14 @@ function(\Ratchet\Client\WebSocket $conn) use ($message, $event_loop, $incomingCallback, $self) { - $conn->on('error', function($error) use ($conn, $self, $message) { - //it's check only on second request - $self->reconnect(); - $self->send($message);//without - 3 iterations + $conn->on('error', function($error) use ($conn, $self, $message, $incomingCallback) { + //@todo reconnect it's check only on second + //without send - 3 iterations + if ($incomingCallback) { + $self->send($message, $incomingCallback); + } else { + $self->send($message); + } }); if (!is_string($message)) { From 313f562220185e2a18045b2a05ba7d8f8283f0a5 Mon Sep 17 00:00:00 2001 From: bag Date: Sun, 19 Jun 2016 02:11:28 +0300 Subject: [PATCH 13/23] fix ws loop reconnect. work only on second request --- .../Module/Connector/RatchetClient.php | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/Classes/Module/Connector/RatchetClient.php b/src/Classes/Module/Connector/RatchetClient.php index 2c3c502..effcce4 100644 --- a/src/Classes/Module/Connector/RatchetClient.php +++ b/src/Classes/Module/Connector/RatchetClient.php @@ -86,27 +86,37 @@ public function close() return $this; } - public function send($message, callable $incomingCallback = null) + public function reconnect() { - //reconnec - //force new connect - $self = $this; + $this->event_loop = null; + $this->connector = null; + $this->promise = null; + Logger::dbg(':WS_RECONNECT:'); + + return $this; + } + + public function send($message, callable $incomingCallback = null) + { + $ratchet_module = $this; $is_first = empty($this->event_loop); $event_loop = $this->getEventLoop(); + $this->getPromise()->then( function(\Ratchet\Client\WebSocket $conn) - use ($message, $event_loop, $incomingCallback, $self) + use ($message, $event_loop, $incomingCallback, $ratchet_module) { - - $conn->on('error', function($error) use ($conn, $self, $message, $incomingCallback) { + $conn->on('error', function($error) use ($conn, $ratchet_module, $message, $incomingCallback) { //@todo reconnect it's check only on second //without send - 3 iterations + Logger::log()->notice("AMQP2WS(service): error"); + $ratchet_module->reconnect(); if ($incomingCallback) { - $self->send($message, $incomingCallback); + $ratchet_module->send($message, $incomingCallback); } else { - $self->send($message); + $ratchet_module->send($message); } }); @@ -119,7 +129,13 @@ function(\Ratchet\Client\WebSocket $conn) ); } if (getenv('FINXLOG_DEBUG')) { - Logger::log()->info("AMQP2WS(service) send try:\t{$message}"); + Logger::log()->info( + "AMQP2WS(service) send try:\t" . ( + getenv('FINXLOG_DEBUG') == \Monolog\Logger::DEBUG + ? preg_replace('~\s+~u', ' ', $message) . '!!!!!' + : substr($message, 0, 70) . '...' + ) + ); if (empty($conn->listeners('close'))) { //new connect $conn->on( 'close', From c8b77be7b3bae63828a5c6e18692c4e72c010f11 Mon Sep 17 00:00:00 2001 From: bag Date: Mon, 20 Jun 2016 03:02:06 +0300 Subject: [PATCH 14/23] front draw graph, command test path, rename: doji to AAPL, full param transport --- command/test/add_amqp_ws_subscribe.php | 17 - command/test/add_amqp_ws_subscribe.sh | 8 - command/test/add_ws_msg_client.php | 29 -- command/test/add_ws_msg_service.php | 35 -- command/test/get_quotation_agg.php | 8 + command/test/ws/add_amqp_ws_subscribe.php | 15 + command/test/ws/add_amqp_ws_subscribe.sh | 7 + command/test/ws/add_ws_msg_client.php | 38 +++ command/test/ws/add_ws_msg_service.php | 43 +++ src/Classes/Model/AbsElasticaModel.php | 2 +- src/Classes/Model/Quotation.php | 4 + src/Classes/Model/QuotationAgg.php | 33 +- .../Module/ClientQueue/SearchQuotation.php | 61 ++-- .../Ratchet/QuotationWebSocketDelivery.php | 72 ++-- tests/ModelQuotationAggTest.php | 4 +- www/index.html | 9 +- www/js/app.js | 310 +++++++++++++----- 17 files changed, 453 insertions(+), 242 deletions(-) delete mode 100755 command/test/add_amqp_ws_subscribe.php delete mode 100755 command/test/add_amqp_ws_subscribe.sh delete mode 100755 command/test/add_ws_msg_client.php delete mode 100755 command/test/add_ws_msg_service.php create mode 100755 command/test/get_quotation_agg.php create mode 100755 command/test/ws/add_amqp_ws_subscribe.php create mode 100755 command/test/ws/add_amqp_ws_subscribe.sh create mode 100755 command/test/ws/add_ws_msg_client.php create mode 100755 command/test/ws/add_ws_msg_service.php diff --git a/command/test/add_amqp_ws_subscribe.php b/command/test/add_amqp_ws_subscribe.php deleted file mode 100755 index fedd122..0000000 --- a/command/test/add_amqp_ws_subscribe.php +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/php -put([ - 'quotation' => 'BTCUSD', - 'agg' => null, - 'agg_period' => null, - ]) - ->put([ - 'quotation' => 'BTCUSD', - 'agg' => 'doji', - 'agg_period' => key((new \FinXLog\Model\QuotationAgg)->getAggPeriod()), - ]); \ No newline at end of file diff --git a/command/test/add_amqp_ws_subscribe.sh b/command/test/add_amqp_ws_subscribe.sh deleted file mode 100755 index 8291f5e..0000000 --- a/command/test/add_amqp_ws_subscribe.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -FINXLOG_ROOT_DIR=`dirname $0`/../../; -#import .env -export $(cat $FINXLOG_ROOT_DIR.env | grep -P '^FINXLOG_(QUOTATION|AMQP)') - -beanstool put -t=$FINXLOG_AMQP_TUBE_WS -b "{\"quotation\":\"BTCUSD\",\"agg\":null,\"agg_period\":null}" - -beanstool put -t=$FINXLOG_AMQP_TUBE_WS -b "{\"quotation\":\"BTCUSD\",\"agg\":\"doji\",\"agg_period\":\"M1\"}" \ No newline at end of file diff --git a/command/test/add_ws_msg_client.php b/command/test/add_ws_msg_client.php deleted file mode 100755 index 037181e..0000000 --- a/command/test/add_ws_msg_client.php +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/php -then(function(\Ratchet\Client\WebSocket $conn) use ($listen_count) { - $conn->on('message', function($msg) use ($conn, $listen_count) { - echo "\nReceived: " . substr($msg, 0, 100). "...\n\n"; - if ($listen_count) { - static $count = 0; - if (++$count >= $listen_count) { - $conn->close(); - } - } - - }); - - $conn->send('{"type": "quotations" }'); - $conn->send('{"type":"subscribe","quotation":"_ALL"}'); - $conn->send('{"type":"subscribe","quotation":"BTCUSD"}'); -}); \ No newline at end of file diff --git a/command/test/add_ws_msg_service.php b/command/test/add_ws_msg_service.php deleted file mode 100755 index d231e0a..0000000 --- a/command/test/add_ws_msg_service.php +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/php -addWsMessage(' - { - "type":"send", - "quotation":"_ALL", - "agg_period":null, - "quotations":[ - {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, - {"S":"TST","T":"2016\/06\/02 04:05:51","B":4} - ] - } -'); - -//slice of real: BTCUSD -$client->addWsMessage(' - { - "type":"send", - "quotation":"BTCUSD", - "agg_period":null, - "quotations":[ - {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151}, - {"S":"BTCUSD","T":"2016\/06\/11 04:05:51","B":568.151} - ] - } -'); - - -$client->addWsMessage(['type' => 'conn_count']); -$client->addWsMessage(['type' => 'all', 'xx' => 123]); -$client->addWsMessage(['type' => 'test']); diff --git a/command/test/get_quotation_agg.php b/command/test/get_quotation_agg.php new file mode 100755 index 0000000..87465ee --- /dev/null +++ b/command/test/get_quotation_agg.php @@ -0,0 +1,8 @@ +#!/usr/bin/php +getAAPL('BTCUSD', 'day') +); \ No newline at end of file diff --git a/command/test/ws/add_amqp_ws_subscribe.php b/command/test/ws/add_amqp_ws_subscribe.php new file mode 100755 index 0000000..0a91e9d --- /dev/null +++ b/command/test/ws/add_amqp_ws_subscribe.php @@ -0,0 +1,15 @@ +#!/usr/bin/php +put([ + 'quotation' => 'BTCUSD' + ]) + ->put([ + 'quotation' => 'BTCUSD', + 'agg_type' => 'AAPL', + 'agg_period' => 'M1', + ]); \ No newline at end of file diff --git a/command/test/ws/add_amqp_ws_subscribe.sh b/command/test/ws/add_amqp_ws_subscribe.sh new file mode 100755 index 0000000..37872fa --- /dev/null +++ b/command/test/ws/add_amqp_ws_subscribe.sh @@ -0,0 +1,7 @@ +#!/bin/bash +FINXLOG_ROOT_DIR=`dirname $0`/../../../; +#import .env +export $(cat $FINXLOG_ROOT_DIR.env | grep -P '^FINXLOG_(QUOTATION|AMQP)') + +beanstool put -t=$FINXLOG_AMQP_TUBE_WS -b '{"quotation":"BTCUSD"}'; +beanstool put -t=$FINXLOG_AMQP_TUBE_WS -b '{"quotation":"BTCUSD","agg_type":"AAPL","agg_period":"M1"}'; diff --git a/command/test/ws/add_ws_msg_client.php b/command/test/ws/add_ws_msg_client.php new file mode 100755 index 0000000..8c0cc66 --- /dev/null +++ b/command/test/ws/add_ws_msg_client.php @@ -0,0 +1,38 @@ +#!/usr/bin/php +then( + function(\Ratchet\Client\WebSocket $conn) + use ($listen_count) + { + $conn->on( + 'message', + function($msg) use ($conn, $listen_count) + { + echo "\nReceived: " . substr($msg, 0, 100). "...\n\n"; + if ($listen_count) { + static $count = 0; + if (++$count >= $listen_count) { + $conn->close(); + } + } + } + ); + + $conn->send('{"type": "quotations" }'); + $conn->send('{"type":"subscribe","quotation":"_ALL"}'); + $conn->send('{"type":"subscribe","quotation":"BTCUSD"}'); + $conn->send('{"type":"subscribe","quotation":"BTCUSD","agg_period": "M1","agg_type":"AAPL"}'); + } + ); \ No newline at end of file diff --git a/command/test/ws/add_ws_msg_service.php b/command/test/ws/add_ws_msg_service.php new file mode 100755 index 0000000..3e99d3b --- /dev/null +++ b/command/test/ws/add_ws_msg_service.php @@ -0,0 +1,43 @@ +#!/usr/bin/php +addWsMessage(' + { + "type":"send", + "quotation":"_ALL", + "quotations":[ + {"S":"BTCUSD","T":"2016/06/11 04:05:51","B":568.151}, + {"S":"TST","T":"2016/06/02 04:05:51","B":4} + ] + } +'); + +$client->addWsMessage(' + { + "type":"send", + "quotation":"BTCUSD", + "quotations":[ + {"S":"BTCUSD","T":"2016/06/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016/06/11 04:05:51","B":568.151} + ] + } +'); + +$client->addWsMessage(' + { + "type":"send", + "quotation":"BTCUSD", + "agg_period":"M1", + "agg_type":"AAPL", + "AAPL":[ + {"S":"BTCUSD","T":"2016/06/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016/06/11 04:05:51","B":568.151} + ] + } +'); + +$client->addWsMessage(['type' => 'conn_count']); +$client->addWsMessage(['type' => 'all', 'xx' => 123]); +$client->addWsMessage(['type' => 'test']); diff --git a/src/Classes/Model/AbsElasticaModel.php b/src/Classes/Model/AbsElasticaModel.php index c2b7800..a87cb36 100644 --- a/src/Classes/Model/AbsElasticaModel.php +++ b/src/Classes/Model/AbsElasticaModel.php @@ -121,7 +121,7 @@ public function checkConnector(Iface\Connector $connector) /** * return * @param Query $query - * @return array + * @return \Elastica\Document[] */ public function getDocuments(Query $query) { diff --git a/src/Classes/Model/Quotation.php b/src/Classes/Model/Quotation.php index 2b9b3cd..3891ac4 100644 --- a/src/Classes/Model/Quotation.php +++ b/src/Classes/Model/Quotation.php @@ -26,6 +26,10 @@ class Quotation extends AbsElasticaModel "sort": [] }'; + /** + * @param string $quotation + * @return \Elastica\Document[] + */ public function getQuotations($quotation = null) { $query = json_decode($this->query_quotations, true); diff --git a/src/Classes/Model/QuotationAgg.php b/src/Classes/Model/QuotationAgg.php index 0ce6761..71ed7c2 100644 --- a/src/Classes/Model/QuotationAgg.php +++ b/src/Classes/Model/QuotationAgg.php @@ -23,7 +23,7 @@ class QuotationAgg extends Quotation "first":{"top_hits":{"size": 1,"sort":[{"T": {"order": "asc"}}]}} w/o deviation: extended_stats => stats */ - private $query_doji = '{ + private $query_agg_period = '{ "query": { "bool": { "must": [ @@ -94,13 +94,11 @@ class QuotationAgg extends Quotation }'; /** - * get DOJI by exchange subject (japanese candlesticks) - * [date][min,max,first,last] * @param string $subject exchange subject(EURUSD, USDBTC) * @param string $interval (period) * @return mixed */ - public function getDoji($subject, $interval = 'day') + public function getAgg($subject, $interval = 'day') { return $this->getAggregations( $this->getDojiQuery( @@ -110,6 +108,29 @@ public function getDoji($subject, $interval = 'day') ); } + /** + * AAPL (doji) historical OHLC data like the Google Finance API + * [date, open, high, low, close] + * @param $subject + * @param string $interval + * @return array + */ + public function getAAPL($subject, $interval = 'M1') + { + $prepared_result = []; + foreach ($this->getAgg($subject, $interval) as $agg) { + $prepared_result[] = [ + 1000 * strtotime($agg['key_as_string']), + (float) $agg['first']['buckets'][0]['avg']['value'], + (float) $agg['stat']['max'], + (float) $agg['stat']['min'], + (float) $agg['last']['buckets'][0]['avg']['value'], + ]; + } + + return $prepared_result; + } + /** * return * @param Query $query @@ -125,8 +146,7 @@ public function getAggregations(Query $query) public function getDojiQuery($subject, $interval = 'day') { - $query = json_decode($this->query_doji, true); - + $query = json_decode($this->query_agg_period, true); $query['query']['bool']['must'][0]['query_string']['query'] = $subject; $query['aggs']['date']['date_histogram']['interval'] = isset($this->agg_period[$interval]) @@ -140,5 +160,4 @@ public function getAggPeriod() { return $this->agg_period; } - } diff --git a/src/Classes/Module/ClientQueue/SearchQuotation.php b/src/Classes/Module/ClientQueue/SearchQuotation.php index 9744ee2..b159693 100644 --- a/src/Classes/Module/ClientQueue/SearchQuotation.php +++ b/src/Classes/Module/ClientQueue/SearchQuotation.php @@ -16,34 +16,49 @@ class SearchQuotation extends AbsQuotation public function makeJob(Job $queue_job) { - $job = json_decode($queue_job->getData(), true); + $job = json_decode($queue_job->getData(), true) + + [ + 'agg_type' => null, + 'agg_period' => null, + ]; if (!is_array($job) || empty($job['quotation'])) { throw new Exception\WrongParams('!job with quotation'); } - if (!empty($job['agg'])) { - $quotations = $this->getModelQuotation() - ->getDoji($job['quotation'], $job['agg_period']); - } else { - $quotations = $this->getModelQuotation() - ->getQuotations( - $job['quotation'] == static::QUOTATION_ALL - ? null - : $job['quotation'] - ); - foreach ($quotations as $key => $value) { - /** - * @var $value Document - */ - $quotations[$key] = $value->getData(); - } + + switch ($job['agg_type']) { + case 'agg': + case 'full': + $quotations = $this->getModelQuotation() + ->getAgg($job['quotation'], $job['agg_period']); + break; + case 'doji': + case 'AAPL': + $quotations = $this->getModelQuotation() + ->getAAPL($job['quotation'], $job['agg_period']); + break; + default: + $quotations = $this->getModelQuotation() + ->getQuotations( + $job['quotation'] == static::QUOTATION_ALL + ? null + : $job['quotation'] + ); + foreach ($quotations as $key => $value) { + $quotations[$key] = $value->getData(); + } + break; } - $this->addWsMessage([ - 'type' => 'send', - 'quotation' => $job['quotation'], - 'agg_period' => $job['agg_period'], - 'quotations' => $quotations, - ]); + + $this->addWsMessage( + [ + 'type' => 'send', + 'quotations' => $quotations, + ] + + $job + ); + + return $this; } public function run() diff --git a/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php b/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php index 8299d93..03158fc 100644 --- a/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php +++ b/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php @@ -117,7 +117,7 @@ public function getPreparedMessage($msg, ConnectionInterface $conn = null) || !is_array($message) || empty($message['type']) ) { - throw new WrongParams("WS: !msg[type]: $msg"); + throw new WrongParams("WS: !msg[type]: " .var_export($msg, true)); } if ( !empty($message['quotation']) @@ -131,11 +131,11 @@ public function getPreparedMessage($msg, ConnectionInterface $conn = null) } if ( - !empty($message['doji']) - && !preg_match('~^[\w\d_\.]+$~u', $message['doji']) + !empty($message['agg_period']) + && !preg_match('~^[\w\d_\.]+$~u', $message['agg_period']) ) { - throw (new WrongParams("WS: wrong doji: $msg")) - ->setParams(['doji']); + throw (new WrongParams("WS: wrong agg_period: $msg")) + ->setParams(['agg_period']); } return $message; @@ -177,6 +177,14 @@ public function onMessage(ConnectionInterface $conn, $msg) //allow service subscribe $this->addClientIncoming($conn, $message); } catch (\Throwable $e) { + $error_message = ['type' => 'error']; + if (getenv('FINXLOG_DEBUG') && getenv('FINXLOG_DEBUG') < 300 /* \Monolog\Logger::NOTICE */) { + $error_message['error'] = [ + 'class' => get_class($e), + 'message' => $e->getMessage(), + ]; + } + $conn->send(json_encode($error_message)); Logger::log()->warning( "WS: {$e->getMessage()} from " . ( @@ -223,7 +231,10 @@ protected function addServiceIncoming(ConnectionInterface $conn, array $message) $this->getSubscribers()->get($message) as $subscribe ) { - $subscribe['ws']->send(json_encode($message, JSON_HEX_TAG)); + $subscribe['ws']->send(json_encode( + ['type' => 'subscribe'] + + $message + )); } Logger::log()->debug(':2js:'); @@ -243,6 +254,27 @@ public function getSubscribers() return $this->subscribers; } + protected function checkPeriod($period, $field_name = null) + { + if ( + strlen($period) + && !is_numeric($period) + && empty((new QuotationAgg)->getAggPeriod()[$period]) + ) { + $e = new WrongParams( + "WS: wrong period: $period" + . ($field_name !== null ? " field: $field_name" : '') + ); + if ($field_name !== null) { + $e->setParams($field_name); + } + + throw $e; + } + + return $this; + } + /** * set client opts * @param ConnectionInterface|RFC6455\Connection $conn @@ -257,42 +289,28 @@ protected function addClientIncoming(ConnectionInterface $conn, array $message) switch ($message['type']) { case 'quotations': - //mock $conn->send(json_encode([ "type" => 'quotations', 'quotations' => ['BTCUSD','USDBTC','USDEUR','EURUSD'] ])); break; case 'subscribe': - if (!empty($message['doji'])) { - $all_period = (new QuotationAgg)->getAggPeriod()[strtoupper($message['doji'])]; - if (empty($all_period[strtoupper($message['doji'])])) { - throw (new WrongParams('WS: wrong doji period on subscribe')) - ->setParams(['doji']); - } - $message['agg_period'] = $all_period[strtoupper($message['doji'])]; + if (!empty($message['agg_period'])) { + $this->checkPeriod($message['agg_period']); } - - /** - * [USDEUR][3600][#123] = [ ... ]; - */ - $this->getSubscribers()->add( - ['ws' => $conn] + $message + ['agg' => null,] - ); + $this->getSubscribers() + ->add(['ws' => $conn] + $message); //load previous period - $this->addJob($new_job = [ - 'quotation' => $message['quotation'], - 'agg' => !empty($message['doji']) ? 'doji' : null, - 'agg_period' => !empty($message['doji']) ? $message['doji'] : null, - ]); - Logger::log()->info('AMQP add:' . json_encode($new_job)); + $this->addJob($message); + Logger::log()->info('AMQP add:' . json_encode($message)); break; case 'unsubscribe': $this->getSubscribers()->drop($conn, $message); Logger::log()->info('AMQP unsubscribe'); break; } + return $this; } diff --git a/tests/ModelQuotationAggTest.php b/tests/ModelQuotationAggTest.php index ff2960d..a9526f3 100644 --- a/tests/ModelQuotationAggTest.php +++ b/tests/ModelQuotationAggTest.php @@ -29,10 +29,10 @@ public function test_query() } - public function test_dogi() + public function test_getAgg() { try { - $this->assertTrue(is_array($this->getApp()->getDoji('BTCUSD'))); + $this->assertTrue(is_array($this->getApp()->getAgg('BTCUSD'))); } catch (\Elastica\Exception\ConnectionException $e) { //connection error } catch (Throwable $e) { diff --git a/www/index.html b/www/index.html index ba0450f..7e3a07c 100644 --- a/www/index.html +++ b/www/index.html @@ -2,11 +2,14 @@ - + + +
- -
+@todo: + +
diff --git a/www/js/app.js b/www/js/app.js index 7f4450b..67498a8 100644 --- a/www/js/app.js +++ b/www/js/app.js @@ -1,17 +1,10 @@ -Highcharts.setOptions({ - global: { - useUTC: false - } -}); - var FinXLog = { + //namespace config: {}, start: function() { - this.ws.setup(); - for (var i = 0; i < this.graph.length; ++i) { - this.graph[i].setup(); - } - this.ws.subscribe('_ALL'); + //this.graph.start(); + this.ws.start(); + this.ws.subscribe('BTCUSD', 'M1', 'AAPL'); }, dbg: function(msg, obj) { if (!FinXLog.config.FINXLOG_DEBUG) return; @@ -19,146 +12,283 @@ var FinXLog = { if (obj) console.log(obj); }, ws: { + conn: null, subscribes: [ - {type: 'subscribe', quotation: '_ALL', doji: null} +// {type: 'subscribe', quotation: '_ALL', doji: null} +// {type: 'subscribe', quotation: 'BTCUSD'} +// {type: 'subscribe', quotation: 'BTCUSD', 'agg_type': 'AAPL', 'agg_period': 'H1'} ], - query_onstart: [//just for non-static + query_onstart: [ + //prepare selector and build table {type: 'quotations'} ], - conn: null, - setup: function () { + start: function (){ this.conn = new WebSocket(this.getWsUrl()); this.conn.onopen = function() { + var i; if (FinXLog.config.FINXLOG_DEBUG) { FinXLog.dbg('WS: open'); } - for (var i = 0; i < FinXLog.ws.query_onstart.length; ++i) { - FinXLog.ws.conn.send(FinXLog.ws.query_onstart[i]); + for (i = 0; i < FinXLog.ws.query_onstart.length; ++i) { + FinXLog.ws.send(FinXLog.ws.query_onstart[i]); } - - for (var i = 0; i < FinXLog.ws.subscribes.length; ++i) { + for (i = 0; i < FinXLog.ws.subscribes.length; ++i) { FinXLog.ws.send(FinXLog.ws.subscribes[i]); } + FinXLog.ws.subscribe('BTCUSD', 'M1', 'AAPL'); }; this.conn.onmessage = this.onmessage; }, + /* + message2graph: function (message) { + var graph_data = {}; + for (var i = 0; i < message.quotations.length; ++i) { + var quotation = message.quotations[i]; + FinXLog.dbg(quotation); + if (typeof graph_data[quotation.S] == 'undefined') { + graph_data[quotation.S] = []; + } + graph_data[quotation.S].push({ + name: quotation.S, + x: new Date(quotation.T).getTime(), + y: quotation.B + }); + } + + return graph_data; + },*/ onmessage: function (request) { var message; try { - message = JSON.parse(request); + message = JSON.parse(request.data); FinXLog.dbg('incoming:'); - FinXLog.dbg(message.data.type); + FinXLog.dbg(message); } catch (err) { FinXLog.dbg('wrong incoming:'); FinXLog.dbg(err); FinXLog.dbg(request.data); - throw err + throw err; } - - switch (message.data.type) { + switch (message.type) { case 'quotations': { - for (var i = 0; i < msg.data.quotations.length; ++i) { - this.dbg(msg.data.quotations[i]); + $(message.quotations).each(function(i, item) { + var opt = document.createElement('option'); + opt.value = item; + opt.appendChild(document.createTextNode(item)) + $('select[name="quotation"]').append(opt) + }); + break; + } + case 'error': { + FinXLog.dbg(message); + + break; + } + case 'subscribe': { + switch(message.agg_type) { + case 'AAPL': { + FinXLog.draw(message.quotations); + } + default: { + FinXLog.dbg('bot ready draw for:', message); + } } break; } + default: { + FinXLog.dbg(message); + } } }, - subscribe: function (subj, doji, stop) { + subscribe: function (subj, agg_period, agg_type, stop) { var request = { 'type': stop ? 'unsubscribe' : 'subscribe', 'quotation': subj, - 'doji': doji ? doji : null + 'agg_period': agg_period, + 'agg_type': agg_type }; + this.send(request); if (stop) { - for (var i = 0; i < FinXLog.ws.subscribes.length; ++i) { + for (var i = 0; i < this.subscribes.length; ++i) { if (JSON.stringify(FinXLog.ws.subscribes[i]) == JSON.stringify(request)) { //@todo //FinXLog.ws.subscribes.remove console.log('remove:'); console.log(FinXLog.ws.subscribes[i]); } - } } else { - FinXLog.ws.subscribes.push(request) + this.subscribes.push(request) } }, send: function (obj) { - FinXLog.dbg('send:', obj); - this.conn.send(JSON.stringify(obj)); + if (this.conn) { + FinXLog.dbg('send:', obj); + this.conn.send(JSON.stringify(obj)); + } else { + FinXLog.dbg('skip send(!conn):', obj); + } }, getWsUrl: function () { return 'ws://' - + (FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_HOST ? FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_HOST : FinXLog.config.FINXLOG_DOMAIN) - + (FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_PORT ? ':' + FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_PORT : '') + + ( + FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_HOST + ? FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_HOST + : FinXLog.config.FINXLOG_DOMAIN + ) + + ( + FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_PORT + ? ':' + FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_PORT + : '' + ) + FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_PATH ; } }, - graph: [ - { - conn: null, - setup: function () { - this.conn = new Highcharts.Chart(this.graph_opts); + graph: { + list: [], + default_list: [ + { + quotation: 'EURUSD', + opts: {}, + conn: null + } + ], + start: function (){ + /*Highcharts.setOptions({ + global: { + useUTC: false + } + });*/ + for (var i = 0; i < FinXLog.graph.default_list.length; ++i) { + var graph_cur = FinXLog.graph.default_list[i]; + graph_cur.opts = this.blank_opts; + graph_cur.opts.title.text = 'Real Time ' + graph_cur.quotation; + graph_cur.opts.yAxis.title.text = 'ratio'; + //graph_cur.conn = new Highcharts.Chart(graph_cur.opts); + FinXLog.graph.list.push(graph_cur); + } + }, + blank_opts: { + title: {text: 'Real Time quotations'}, + xAxis: { + type: 'datetime', + tickPixelInterval: 100 }, - graph_opts: { - title: {text: 'Real Time Samples'}, - xAxis: { - type: 'datetime', - tickPixelInterval: 100 - }, - yAxis: { - title: {text: 'Samples'}, - tickInterval: 10, - min: 0, - max: 100 - }, - tooltip: { - formatter: function () { - return '' + this.series.name + '
' - + JSON.stringify([ - Highcharts.dateFormat('%Y-%m-%d %H:%M:%S', this.x), - this.y - ]); - } - }, - chart: { - type: 'spline', + yAxis: { + title: {text: 'Samples'}, + tickInterval: 10, + min: 0, + max: 100 + }, + tooltip: { + formatter: function () { + return '' + this.series.name + '
' + + JSON.stringify([ + Highcharts.dateFormat('%Y-%m-%d %H:%M:%S', this.x), + this.y + ]); + } + }, + chart: { + type: 'spline', renderTo: 'graph_container', events: { - load: function () { - if (FinXLog.config.FINXLOG_DEBUG) { - console.log('chart load'); - } - } + load: function () { + FinXLog.dbg('chart load'); } - }, - series: [{ - name: 'tst', - data: (function () { - var data = [], - time = (new Date()).getTime(), - i; - for (i = -19; i <= 0; i++) { - data.push({ - x: time + (i * 1000), - y: 0 - }); - } - return data; - })() - }], - draw: function (data) { - console.log('draw:'); - console.log(data); } - } + }, + series: [] } - ] + }, + draw: function (data) { + + // split the data set into ohlc and volume + var ohlc = [], + volume = [], + dataLength = data.length, + // set the allowed units for data grouping + groupingUnits = [[ + 'week', // unit name + [1] // allowed multiples + ], [ + 'month', + [1, 2, 3, 4, 6] + ]], + i = 0; + for (i; i < dataLength; i += 1) { + ohlc.push([ + data[i][0], // the date + data[i][1], // open + data[i][2], // high + data[i][3], // low + data[i][4] // close + ]); + + volume.push([ + data[i][0], // the date + data[i][5] // the volume + ]); + } + + + // create the chart + $('#container').highcharts('StockChart', { + + rangeSelector: { + selected: 1 + }, + + title: { + text: 'AAPL Historical' + }, + + yAxis: [{ + labels: { + align: 'right', + x: -3 + }, + title: { + text: 'OHLC' + }, + height: '60%', + lineWidth: 2 + }, { + labels: { + align: 'right', + x: -3 + }, + title: { + text: 'Volume' + }, + top: '65%', + height: '35%', + offset: 0, + lineWidth: 2 + }], + + series: [{ + type: 'candlestick', + name: 'AAPL', + data: ohlc, + dataGrouping: { + units: groupingUnits + } + }, { + type: 'column', + name: 'Volume', + data: volume, + yAxis: 1, + dataGrouping: { + units: groupingUnits + } + }] + }); + } }; includeJS('js/config_autobuild.js'); @@ -169,4 +299,4 @@ function includeJS(url) script.src = url; script.type = 'text/javascript'; document.getElementsByTagName('head')[0].appendChild(script); -} \ No newline at end of file +} From 03f16c730b0a3267258ee60f23d348a0eb422435 Mon Sep 17 00:00:00 2001 From: bag Date: Mon, 20 Jun 2016 03:07:07 +0300 Subject: [PATCH 15/23] readme --- readme.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/readme.md b/readme.md index 11e58d3..1c23382 100644 --- a/readme.md +++ b/readme.md @@ -5,6 +5,7 @@ Complete: - save - queue - elastic log + - highcharts AAPL with websocket Instruments: - ElasticSearch for quick big data search @@ -16,13 +17,12 @@ Optional: - BeanstalkD(AMQP Queue manager). reason for use: quick delivery for "any" load with single stream(bash-scrpit) @todo - - highcharts on websocket - simple json API + - switch quottion and period +maby: - Ratchet + WAMP + ZMQ -Optional: - sql db - - make quotation_exchange2db.sh - - lock for parallel import + - split traffic for parallel import ( % n ) # Install ```bash From 88755370f131964925f099ff25c527e11db0b481 Mon Sep 17 00:00:00 2001 From: BAGArt Date: Mon, 20 Jun 2016 03:08:23 +0300 Subject: [PATCH 16/23] Update readme.md --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 1c23382..ee0178a 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,4 @@ -v 0.9.1 +v 0.9.2 Complete: - import - filter From 4320d6e1d88b5919c3f18ca5753a637a33b7d7a0 Mon Sep 17 00:00:00 2001 From: BAGArt Date: Wed, 22 Jun 2016 23:17:01 +0300 Subject: [PATCH 17/23] Update readme.md --- readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index ee0178a..352d62b 100644 --- a/readme.md +++ b/readme.md @@ -7,7 +7,7 @@ Complete: - elastic log - highcharts AAPL with websocket -Instruments: +Components: - ElasticSearch for quick big data search - Composer, PSR-4 - Monolog @@ -19,6 +19,7 @@ Optional: @todo - simple json API - switch quottion and period + maby: - Ratchet + WAMP + ZMQ - sql db From 8302b0420d7922039d77d678c1886adf0b9727af Mon Sep 17 00:00:00 2001 From: BAGArt Date: Wed, 22 Jun 2016 23:18:11 +0300 Subject: [PATCH 18/23] Update readme.md --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 352d62b..7966ad4 100644 --- a/readme.md +++ b/readme.md @@ -64,8 +64,8 @@ important: direct import has minimal guarantee for stable: AMQP is depend by high performance, scalable beanstalk (and opensource client) -## Required(curently - ): WebSocket -WebSocket is require AMQP +## Required(currently - ): WebSocket +AMQP is require for WebSocket or need implement async elasticsearch client with guzzle or reactphp/http-client or need implement async reactphp/child-process not ready but simple: From 29192bf3852e2216adbd556ec890b2bf22423526 Mon Sep 17 00:00:00 2001 From: BAGArt Date: Fri, 24 Jun 2016 13:32:14 +0300 Subject: [PATCH 19/23] Update readme.md --- readme.md | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/readme.md b/readme.md index 7966ad4..ae12336 100644 --- a/readme.md +++ b/readme.md @@ -1,29 +1,30 @@ v 0.9.2 -Complete: - - import - - filter - - save - - queue - - elastic log - - highcharts AAPL with websocket +Quotation Exchange graph builder -Components: - - ElasticSearch for quick big data search +Front End: + - static html + - websocket + - HighCharts (Doji graph) + +Back End: + - ReactPHP (Ratchet for WebSocket) + - ElasticSearch for big data aggregation + - BeanstalkD Queue + - import (bash or php) - Composer, PSR-4 - Monolog - ".ENV" environment -Optional: - - BeanstalkD(AMQP Queue manager). reason for use: quick delivery for "any" load with single stream(bash-scrpit) - @todo - - simple json API - - switch quottion and period + - simple json API + - visual switch for quotation and period + - load real exchange + - performance tests -maby: +maybe: - Ratchet + WAMP + ZMQ - - sql db - - split traffic for parallel import ( % n ) + - node.js ws loop + - for too big import stream: split traffic(X % n) for parallel import in different instance with queue controll # Install ```bash From e2d59c88e96744eaafdf41f3efae9ada2cc388d2 Mon Sep 17 00:00:00 2001 From: BAGArt Date: Fri, 24 Jun 2016 13:33:10 +0300 Subject: [PATCH 20/23] Update readme.md --- readme.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index ae12336..d6f035b 100644 --- a/readme.md +++ b/readme.md @@ -1,5 +1,8 @@ -v 0.9.2 -Quotation Exchange graph builder +# Quotation Exchange view, graph builder + +Master: v0.9.2 + + Front End: - static html From 94e209b6bf5bbe4d3c873cc24d2f35eba79ac8d1 Mon Sep 17 00:00:00 2001 From: BAGArt Date: Wed, 9 Nov 2016 15:36:13 +0300 Subject: [PATCH 21/23] Update .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index ef4215d..d247c65 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .idea/ vendor/ -composer.lock .env www/js/config_autobuild.js From 9fbe2f7b063af29f82af50e9c8acf74f2a14c525 Mon Sep 17 00:00:00 2001 From: BAGArt Date: Mon, 28 Nov 2016 15:52:57 +0300 Subject: [PATCH 22/23] Create LICENSE.txt --- LICENSE.txt | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..906d5ce --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,20 @@ + Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From 3837fc55f1d8ec8ff033c5f8fb0d542dba0db40d Mon Sep 17 00:00:00 2001 From: BAGArt Date: Fri, 2 Dec 2016 10:05:00 +0300 Subject: [PATCH 23/23] Update LICENSE.txt --- LICENSE.txt | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 906d5ce..f70b712 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,20 +1,21 @@ - Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +The MIT License (MIT) + +Copyright 2016 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE.