Extend text length in statement table

chatterbot/constants.py

@@ -4,11 +4,10 @@
 
 '''
 The maximum length of characters that the text of a statement can contain.
-The number 255 is used because that is the maximum length of a char field
-in most databases. This value should be enforced on a per-model basis by
-the data model for each storage adapter.
+The number 1000 is used to accommodate longer statements and avoid truncation errors.
+This value should be enforced on a per-model basis by the data model for each storage adapter.
 '''
-STATEMENT_TEXT_MAX_LENGTH = 255
+STATEMENT_TEXT_MAX_LENGTH = 1000
 
 '''
 The maximum length of characters that the text label of a conversation can contain.
@@ -25,4 +24,5 @@
 # The maximum length of characters that the name of a tag can contain
 TAG_NAME_MAX_LENGTH = 50
 
+# The default name of the Django app used by ChatterBot
 DEFAULT_DJANGO_APP_NAME = 'django_chatterbot'

chatterbot/ext/django_chatterbot/abstract_models.py

@@ -49,11 +49,11 @@ class AbstractBaseStatement(models.Model, StatementMixin):
     """
 
     text = models.CharField(
-        max_length=constants.STATEMENT_TEXT_MAX_LENGTH
+        max_length=1000
     )
 
     search_text = models.CharField(
-        max_length=constants.STATEMENT_TEXT_MAX_LENGTH,
+        max_length=1000,
         blank=True
     )
 
@@ -92,6 +92,15 @@ class AbstractBaseStatement(models.Model, StatementMixin):
 
     class Meta:
         abstract = True
+        indexes = [
+            models.Index(fields=['text']),
+            models.Index(fields=['search_text']),
+            models.Index(fields=['conversation']),
+            models.Index(fields=['created_at']),
+            models.Index(fields=['in_response_to']),
+            models.Index(fields=['search_in_response_to']),
+            models.Index(fields=['persona']),
+        ]
 
     def __str__(self):
         if len(self.text.strip()) > 60:
@@ -114,3 +123,16 @@ def add_tags(self, *tags):
         """
         for _tag in tags:
             self.tags.get_or_create(name=_tag)
+
+    def clean(self):
+        """
+        Custom validation method to ensure data integrity and consistency.
+        """
+        if len(self.text) > constants.STATEMENT_TEXT_MAX_LENGTH:
+            raise ValueError(f'Text length exceeds {constants.STATEMENT_TEXT_MAX_LENGTH} characters.')
+        if len(self.search_text) > constants.STATEMENT_TEXT_MAX_LENGTH:
+            raise ValueError(f'Search text length exceeds {constants.STATEMENT_TEXT_MAX_LENGTH} characters.')
+        if len(self.in_response_to or '') > constants.STATEMENT_TEXT_MAX_LENGTH:
+            raise ValueError(f'In response to length exceeds {constants.STATEMENT_TEXT_MAX_LENGTH} characters.')
+        if len(self.search_in_response_to or '') > constants.STATEMENT_TEXT_MAX_LENGTH:
+            raise ValueError(f'Search in response to length exceeds {constants.STATEMENT_TEXT_MAX_LENGTH} characters.')

chatterbot/ext/django_chatterbot/migrations/0019_extend_text_length.py

@@ -0,0 +1,25 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    """
+    This migration updates the column length for 'text' and 'search_text'
+    in the 'statement' table to 1000 characters.
+    """
+
+    dependencies = [
+        ('django_chatterbot', '0018_text_max_length'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            sql=[
+                "ALTER TABLE statement ALTER COLUMN text TYPE varchar(1000);",
+                "ALTER TABLE statement ALTER COLUMN search_text TYPE varchar(1000);"
+            ],
+            reverse_sql=[
+                "ALTER TABLE statement ALTER COLUMN text TYPE varchar(255);",
+                "ALTER TABLE statement ALTER COLUMN search_text TYPE varchar(255);"
+            ]
+        ),
+    ]