Merge branch 'staging' into gen_ml_flag

Author: paxcema

assets/contributions-agreement/signatures/cla.json

@@ -5119,6 +5119,30 @@
       "created_at": "2023-06-14T06:24:39Z",
       "repoId": 143328315,
       "pullRequestNo": 6587
+    },
+    {
+      "name": "akj2018",
+      "id": 43956935,
+      "comment_id": 1593813121,
+      "created_at": "2023-06-15T22:44:19Z",
+      "repoId": 143328315,
+      "pullRequestNo": 6596
+    },
+    {
+      "name": "truskovskiyk",
+      "id": 7893705,
+      "comment_id": 1594494194,
+      "created_at": "2023-06-16T10:55:59Z",
+      "repoId": 143328315,
+      "pullRequestNo": 6620
+    },
+    {
+      "name": "Aman123lug",
+      "id": 94223645,
+      "comment_id": 1594700571,
+      "created_at": "2023-06-16T13:44:38Z",
+      "repoId": 143328315,
+      "pullRequestNo": 6546
     }
   ]
 }

docs/data-integrations/datastax.mdx

@@ -16,6 +16,10 @@ The required arguments to establish a connection are as follows:
 * `user` is the user to authenticate.
 * `password` is the password to authenticate the user.
 * `secure_connection_bundle` is the path to the `secure_connection_bundle` zip file.
+<Tip>
+If you installed MindsDB locally via pip, you need to install all handler dependencies manually. To do so, go to the handler's folder (mindsdb/integrations/handlers/datastax_handler) and run this command: `pip install -r requirements.txt`.
+</Tip>
+
 
 ## Usage
 

docs/manual-qa/hf_toxicity_tutorial_test.mdx

@@ -0,0 +1,37 @@
+## Testing: Hugging Face - Toxicity Tutorial
+
+Testing CREATE MODEL
+
+```sql
+CREATE MODEL mindsdb.hf_toxicity
+PREDICT PRED
+USING
+engine = 'huggingface',
+task = 'text-classification',
+model_name = 'SkolkovoInstitute/roberta_toxicity_classifier',
+input_column = 'text';
+```
+
+Output:
+
+```sql
+| NAME | ENGINE | PROJECT | ACTIVE | VERSION | STATUS | ACCURACY | PREDICT | UPDATE_STATUS | MINDSDB_VERSION | ERROR | SELECT_DATA_QUERY | TRAINING_OPTIONS | TAG |
+| ---- | ------ | ------- | ------ | ------- | ------ | -------- | ------- | ------------- | --------------- | ----- | ----------------- | ---------------- | --- |
+| hf_toxicity | huggingface | mindsdb | true | 1 | generating | [NULL] | PRED | up_to_date | 23.6.3.1 | [NULL] | [NULL] | {'target': 'PRED', 'using': {'task': 'text-classification', 'model_name': 'SkolkovoInstitute/roberta_toxicity_classifier', 'input_column': 'text'}} | [NULL] |
+```
+
+Testing SELECT (making predictions)
+
+```sql
+SELECT *
+FROM mindsdb.hf_toxicity
+WHERE text = 'I like you. I love you.';
+```
+
+Output:
+
+```sql
+| PRED | PRED_explain | text |
+| ---- | ------------ | ---- |
+| neutral | {"neutral":0.9999547004699707,"toxic":0.000045352782763075083} | I like you. I love you. |
+```

mindsdb/api/http/namespaces/analysis.py

@@ -1,5 +1,6 @@
 import time
 
+import pandas as pd
 from flask import request
 from flask_restx import Resource
 from pandas.core.frame import DataFrame
@@ -17,6 +18,17 @@
 def analyze_df(df: DataFrame) -> dict:
     if len(df) == 0:
         return {}
+
+    cols = pd.Series(df.columns)
+
+    # https://stackoverflow.com/questions/24685012/pandas-dataframe-renaming-multiple-identically-named-columns
+    for dup in cols[cols.duplicated()].unique():
+        cols[cols[cols == dup].index.values.tolist()] = [dup + '.' + str(i) if i != 0 else dup for i in
+                                                         range(sum(cols == dup))]
+
+    # rename the columns with the cols list.
+    df.columns = cols
+
     analysis = analyze_dataset(df)
     return analysis.to_dict()
 

mindsdb/api/mysql/mysql_proxy/classes/sql_query.py

@@ -106,6 +106,12 @@ def get_table_alias(table_obj, default_db_name):
         else:
             name = table_obj.alias.parts[0]
         name = (default_db_name, name)
+    elif isinstance(table_obj, Join):
+        # get from first table
+        return get_table_alias(table_obj.left, default_db_name)
+    else:
+        # unknown yet object
+        return default_db_name, 't', 't'
 
     if table_obj.alias is not None:
         name = name + ('.'.join(table_obj.alias.parts),)

mindsdb/integrations/handlers/gmail_handler/README.md

@@ -9,14 +9,14 @@ To see how the Gmail handler is used, let's walk through the steps to create a s
 
 ## Connect to the Gmail API
 
-To use the Gmail API we need to setup a Google Cloud Project and a Google Account with Gmail enabled.
+To use the Gmail API we need to set up a Google Cloud Project and a Google Account with Gmail enabled.
 
 Before proceeding further, we will need to enable the Gmail API from the Google Cloud Console.
 
-We will also need to create OAuth Client Ids for authenticating users, and possibly an Auth Consent Screen (if this is the first time we're setting up OAuth)
+We will also need to create OAuth Client Ids for authenticating users, and possibly an Auth Consent Screen (if this is the first time we're setting up OAuth).
 
 Setting up OAuth Client Id will give us a credentials file which we will need in our mindsdb setup. You can find more information on how to do
-this [here](https://developers.google.com/gmail/quickstart/python).
+this [here](https://developers.google.com/gmail/api/quickstart/python).
 
 **Optional:**  The credentials file can be stored in the gmail_handler folder in
 the `mindsdb/integrations/handlers/gmail_handler` directory.
@@ -33,7 +33,7 @@ parameters = {
 This creates a database called mindsdb_gmail. This database ships with a table called emails that we can use to search for
 emails as well as to write emails.
 
-You can also create a database by giving the credentials file from a s3 pre signed url.To do this you need to pass in the credentials_file parameter as a signed url.For example:
+You can also create a database by giving the credentials file from a s3 pre signed url. To do this you need to pass in the credentials_file parameter as a signed url. For example:
 
 ~~~~sql
 CREATE DATABASE mindsdb_gmail
@@ -59,7 +59,7 @@ LIMIT 20;
 ~~~~
 This will search your Gmail inbox for any email which contains the text `alert` and is from `google.com` domain (notice the use of the wildcard `*`).
 
-The returned result should have ROWs like this
+The returned result should have ROWs like this,
 
 | id | message_id | thread_id | label_ids | sender | to | date | subject | snippet | history_id | size_estimate | body | attachments |
 | ----------- | ----------- | ----------- | ----------- | ----------- | ----------- | ----------- | ----------- | ----------- | ----------- | ----------- | ----------- | ----------- |
@@ -68,7 +68,7 @@ The returned result should have ROWs like this
 where
 * query - The search term. The query parameter supports all the search terms we can use with gmail. For more details please check [this link](https://support.google.com/mail/answer/7190)
 * label_ids - A comma separated string of labels to search for. E.g. "INBOX,UNREAD" will search for unread emails in inbox, "SENT" will search for emails in the sent folder.
-* include_spam_trash - BOOLEAN (TRUE / FALSE). By default it is FALSE. If included, the search will cover the SPAM and TRASH folders.
+* include_spam_trash - BOOLEAN (TRUE / FALSE). By default, it is FALSE. If included, the search will cover the SPAM and TRASH folders.
 
 ## Writing Emails
 
@@ -112,7 +112,7 @@ USING
   input_column = 'text_spammy',        
   labels = ['ham', 'spam'];
 ~~~~
-* Then you can have to create a view of the email table that contains the snippet or the body of the email.For example by using the snippet:
+* Then you can have to create a view of the email table that contains the snippet or the body of the email. For example by using the snippet:
 ~~~~sql
 CREATE VIEW mindsdb.emails_text AS(
     SELECT snippet AS text_spammy

mindsdb/integrations/handlers/gmail_handler/gmail_handler.py

@@ -12,6 +12,7 @@
 from mindsdb_sql.parser import ast
 from mindsdb.utilities import log
 from mindsdb_sql import parse_sql
+from mindsdb.interfaces.storage.model_fs import HandlerStorage
 
 import os
 import time
@@ -215,6 +216,8 @@ def __init__(self, name=None, **kwargs):
         self.service = None
         self.is_connected = False
 
+        self.storage = HandlerStorage(kwargs['integration_id'])
+
         emails = EmailsTable(self)
         self.emails = emails
         self._register_table('emails', emails)
@@ -239,7 +242,7 @@ def create_connection(self) -> object:
         creds = None
 
         # Get the current dir, we'll check for Token & Creds files in this dir
-        curr_dir = os.path.dirname(__file__)
+        curr_dir = self.storage.folder_get('config')
 
         token_file = os.path.join(curr_dir, 'token.json')
         creds_file = os.path.join(curr_dir, 'creds.json')
@@ -260,6 +263,7 @@ def create_connection(self) -> object:
         with open(token_file, 'w') as token:
             token.write(creds.to_json())
 
+        self.storage.folder_sync('config')
         return build('gmail', 'v1', credentials=creds)
 
     def connect(self) -> object:

requirements/requirements.txt

@@ -19,7 +19,7 @@ sentry-sdk
 walrus==0.8.2
 flask-compress >= 1.0.0
 appdirs >= 1.0.0
-mindsdb-sql >= 0.6.4, < 0.7.0
+mindsdb-sql >= 0.6.5, < 0.7.0
 mindsdb-evaluator >= 0.0.7, < 0.1.0
 checksumdir >= 1.2.0
 duckdb == 0.6.0