Add V1 endpoints for factory, machine, machine_type

smsdk/Auth/auth.py

@@ -3,6 +3,7 @@
 import warnings
 
 from requests.utils import cookiejar_from_dict, default_headers
+import pandas as pd
 
 try:
     import importlib.resources as pkg_resources
@@ -158,11 +159,12 @@ def logout(self):
 
     def check_auth(self):
         """
-        Determine if SDK has access to the client by checking the Cycle API.
+        Determine if SDK has access to the client by checking the Machine API.
         """
         try:
-            url = "{}{}".format(self.host, ENDPOINTS["Cycle"]["alt_url"])
-            resp = self._get_records(url, _limit=1, _only=["_id"])
+            url = "{}{}".format(self.host, ENDPOINTS["Machine"]["url_v1"])
+            resp = self._get_records_mongo_v1(url, limit=1, select=["id"])
+            resp = list(pd.concat(list(resp)).reset_index().values)
             return isinstance(resp, list) and "error" not in resp
         except Exception:  # pylint:disable=broad-except
             return False

smsdk/client.py

@@ -11,7 +11,7 @@
 except ImportError:
     from pandas.io.json import json_normalize
 
-from smsdk.utils import get_url
+from smsdk.utils import get_url, escape_mongo_field_name
 from smsdk.Auth.auth import Authenticator, X_SM_DB_SCHEMA
 from smsdk.tool_register import smsdkentities
 from smsdk.client_v0 import ClientV0
@@ -46,7 +46,7 @@ def dict_to_df(data, normalize=True):
                 # machine type stats are list
                 cols = [*data[0]]
                 cols.remove('stats')
-                df = json_normalize(data, 'stats', cols, record_prefix='stats.')
+                df = json_normalize(data, 'stats', cols, record_prefix='stats.', errors='ignore')
         else:
             try:
                 df = json_normalize(data)
@@ -62,9 +62,12 @@ def dict_to_df(data, normalize=True):
 
         if 'id' in df.columns:
             df.set_index('id', inplace=True)
-
+            
     return df
 
+def generator_to_df(generator) -> pd.DataFrame:
+    data = pd.concat([page for page in generator])
+    return data
 
 # We don't have a downtime schema, so hard code one
 downmap = {'machine__source': 'Machine',
@@ -157,13 +160,17 @@ def get_data_v1(self, ename, util_name, normalize=True, *args, **kwargs):
             # dict params strictly follow {'key':'value'} format
 
             # sub_kwargs = kwargs
-            if util_name in ['get_cycles', 'get_downtime', 'get_parts']:
+            if util_name in ['get_cycles', 'get_downtime', 'get_parts', 'get_factories', 'get_machines', 'get_machine_types']:
                 sub_kwargs = [kwargs]
             else:
                 sub_kwargs = self.fix_only(kwargs)
 
             if len(sub_kwargs) == 1:
-                data = dict_to_df(getattr(cls, util_name)(*args, **sub_kwargs[0]), normalize)
+                if util_name in ['get_factories', 'get_machines', 'get_machine_types']:
+                    # data = dict_to_df(getattr(cls, util_name)(*args, **sub_kwargs[0]), normalize)
+                    return getattr(cls, util_name)(normalize, *args, **sub_kwargs[0])
+                else:
+                    data = dict_to_df(getattr(cls, util_name)(*args, **sub_kwargs[0]), normalize)
             else:
                 data = dict_to_df(getattr(cls, util_name)(*args, **sub_kwargs[0]), normalize)
                 for sub in sub_kwargs[1:]:
@@ -173,7 +180,7 @@ def get_data_v1(self, ename, util_name, normalize=True, *args, **kwargs):
                     data.drop(joined_cols, axis=1)
 
             # To keep consistent, rename columns back from '.' to '__'
-            data.columns = [name.replace('.', '__') for name in data.columns]
+            data.columns = [escape_mongo_field_name(name)for name in data.columns]
 
         else:
             # raise error if requested for unregistered utility
@@ -290,6 +297,109 @@ def get_fields_of_machine_type(self, machine_type, types=[], show_hidden=False,
 
         return fields
 
+
+    def _get_factories(self, *args, normalize=True, **kwargs):
+        """
+        Get list of factories and associated metadata.  Note this includes extensive internal metadata.  
+
+        :param normalize: Flatten nested data structures
+        :type normalize: bool
+        :return: pandas dataframe
+        """
+        return self.get_data_v1('factory_v1', 'get_factories', normalize, *args, **kwargs)
+
+    def _get_machines(self, *args, normalize=True, **kwargs) -> pd.DataFrame:
+        """
+        Get list of machines and associated metadata.  Note this includes extensive internal metadata.  If you only want to get a list of machine names
+        then see also get_machine_names(). 
+
+        :param normalize: Flatten nested data structures
+        :type normalize: bool
+        :return: pandas dataframe
+        """
+        return self.get_data_v1('machine_v1', 'get_machines', normalize, *args, **kwargs)
+
+    def _get_machine_types(self, *args, normalize=True, **kwargs):
+        """
+        Get list of machine types and associated metadata.  Note this includes extensive internal metadata.  If you only want to get a list of machine type names
+        then see also get_machine_type_names(). 
+
+        :param normalize: Flatten nested data structures
+        :type normalize: bool
+        :return: pandas dataframe
+        """
+
+        return self.get_data_v1('machine_type_v1', 'get_machine_types', normalize, *args, **kwargs)
+
+    def get_factories(self, *args, normalize=True, **kwargs):
+        generator = self._get_factories(normalize=normalize, *args, **kwargs)
+        data = generator_to_df(generator)
+        return data
+
+    def get_machines(self, *args, normalize=True, **kwargs):
+        generator = self._get_machines(normalize=normalize, *args, **kwargs)
+        data = generator_to_df(generator)
+        return data
+
+    def get_machine_types(self, *args, normalize=True, **kwargs):
+        generator = self._get_machine_types(normalize=normalize, *args, **kwargs)
+        data = generator_to_df(generator)
+        return data
+
+    def get_machine_names(self, source_type=None, clean_strings_out=True):
+        """
+        Get a list of machine names.  This is a simplified version of get_machines().  
+
+        :param source_type: filter the list to only the specified source_type
+        :type source_type: str
+        :param clean_strings_out: If true, return the list using the UI-based display names.  If false, the list contains the Sight Machine internal machine names.
+        :return: list
+        """
+
+        query_params = {
+            'select': ['source', 'source_clean', 'source_type'], 
+            'order_by': [{'name':'source_clean'}]
+        }
+
+        if source_type:
+            # Double check the type
+            mt = self.get_machine_types(source_type=source_type)
+            # If it was found, then no action to take, otherwise try looking up from clean string
+            mt = self.get_machine_types(source_type_clean=source_type) if not len(mt) else []
+            if len(mt):
+                source_type = mt['source_type'].iloc[0]
+            else:
+                log.error('Machine Type not found')
+                return []
+
+            query_params['source_type'] = source_type
+
+        machines = self.get_data_v1('machine_v1', 'get_machines', normalize=True, **query_params)
+        machines = generator_to_df(machines)
+
+        if clean_strings_out:
+            return machines['source_clean'].to_list()
+        else:
+            return machines['source'].to_list()
+
+    def get_machine_type_names(self, clean_strings_out=True):
+        """
+        Get a list of machine type names.  This is a simplified version of get_machine_types().  
+
+        :param clean_strings_out: If true, return the list using the UI-based display names.  If false, the list contains the Sight Machine internal machine types.
+        :return: list
+        """
+        query_params = {
+            'select': ['source_type', 'source_type_clean'],
+            'order_by': [{'name':'source_type_clean'}]
+        }
+        machine_types = self.get_data_v1('machine_type_v1', 'get_machine_types', normalize=True, **query_params)
+        machine_types = generator_to_df(machine_types)
+
+        if clean_strings_out:
+            return machine_types['source_type_clean'].to_list()
+        else:
+            return machine_types['source_type'].to_list()
     def get_cookbooks(self, **kwargs):
         """
         Gets all of the cookbooks accessable to the logged in user.

smsdk/client_v0.py

@@ -15,7 +15,7 @@
 except ImportError:
     from pandas.io.json import json_normalize
 
-from smsdk.utils import get_url
+from smsdk.utils import get_url, escape_mongo_field_name
 from smsdk.Auth.auth import Authenticator
 from smsdk.tool_register import smsdkentities
 
@@ -250,7 +250,7 @@ def get_data(self, ename, util_name, normalize=True, *args, **kwargs):
                     data.drop(joined_cols, axis=1)
 
             # To keep consistent, rename columns back from '.' to '__'
-            data.columns = [name.replace('.', '__') for name in data.columns]
+            data.columns = [escape_mongo_field_name(name) for name in data.columns]
 
         else:
             # raise error if requested for unregistered utility
@@ -437,11 +437,11 @@ def inner(self, machine_source, types=[], show_hidden=False, return_mtype=False,
                         log.error(f'Unable to find machine type for {machine_source}')
                         return
             try:
-                stats = self.get_machine_types(normalize=False, _limit=1, source_type=machine_type)['stats'][0]
+                stats = self.get_machine_types(normalize=False, _limit=1, source_type=machine_type)
             except KeyError:
                 # explicitly embed string to machine type names esp JCP
                 machine_type = f"'{machine_type}'"
-                stats = self.get_machine_types(normalize=False, _limit=1, source_type=machine_type)['stats'][0]
+                stats = self.get_machine_types(normalize=False, _limit=1, source_type=machine_type)
             except Exception as ex:
                 print(f"Exception in getting machine type stats {ex}")
             kwargs['stats'] = stats
@@ -579,7 +579,7 @@ def get_downtimes_with_cycles(self, normalize=True, clean_strings_in=True, clean
 
         return merged
 
-    def get_factories(self, normalize=True, *args, **kwargs):
+    def get_factories(self, *args, normalize=True, **kwargs):
         """
         Get list of factories and associated metadata.  Note this includes extensive internal metadata.  
 
@@ -589,7 +589,7 @@ def get_factories(self, normalize=True, *args, **kwargs):
         """
         return self.get_data('factory', 'get_factories', normalize, *args, **kwargs)
 
-    def get_machines(self, normalize=True, *args, **kwargs):
+    def get_machines(self, *args, normalize=True, **kwargs) -> pd.DataFrame:
         """
         Get list of machines and associated metadata.  Note this includes extensive internal metadata.  If you only want to get a list of machine names
         then see also get_machine_names(). 
@@ -617,13 +617,12 @@ def get_machine_names(self, source_type=None, clean_strings_out=True):
             # Double check the type
             mt = self.get_machine_types(source_type=source_type)
             # If it was found, then no action to take, otherwise try looking up from clean string
-            if not len(mt):
-                mt = self.get_machine_types(source_type_clean=source_type)
-                if len(mt):
-                    source_type = mt['source_type'].iloc[0]
-                else:
-                    log.error('Machine Type not found')
-                    return []
+            mt = self.get_machine_types(source_type_clean=source_type) if not len(mt) else []
+            if len(mt):
+                source_type = mt['source_type'].iloc[0]
+            else:
+                log.error('Machine Type not found')
+                return []
 
             query_params['source_type'] = source_type
 
@@ -675,7 +674,7 @@ def get_machine_timezone(self, machine_source):
 
         return timezone
 
-    def get_machine_types(self, normalize=True, *args, **kwargs):
+    def get_machine_types(self, *args, normalize=True, **kwargs):
         """
         Get list of machine types and associated metadata.  Note this includes extensive internal metadata.  If you only want to get a list of machine type names
         then see also get_machine_type_names(). 

smsdk/config/api_endpoints.json

@@ -3,27 +3,23 @@
     "url": "/auth/password/login"
   },
   "Cycle": {
-    "url_v1": "/v1/datatab/cycle",
-    "url": "/api/cycle",
-    "alt_url": "/api/cycle"
+    "url_v1": "/v1/datatab/cycle"
   },
   "Factory": {
-    "url" : "/api/factory"
+    "url_v1" : "/v1/obj/factory"
   },
   "MachineType": {
-    "url" : "/api/machinetype",
+    "url_v1" : "/v1/obj/machine_type",
     "fields": "/v1/selector/datatab/cycle/{}/field"
   },
   "Machine": {
-    "url" : "/api/machine"
+    "url_v1" : "/v1/obj/machine"
   },
   "Parts": {
-    "url" : "/api/part",
     "url_v1": "/v1/datatab/part",
     "part_schema": "/v1/obj/part_type"
   },
   "Downtime": {
-    "url" : "/api/downtime",
     "url_v1" : "/v1/datatab/downtime"
 
   },