Python Forum
error in database for chatbot
Thread Rating:
  • 0 Vote(s) - 0 Average
  • 1
  • 2
  • 3
  • 4
  • 5
error in database for chatbot
#1
i'm currently trying to follow a tutorial about coding a chatbot with python. i'm new to python and coding in general except for some projects i did in gamemaker(lol). i'm getting an error when trying to run my code and have difficulties figuring out what the problem is. any help would be greatly appreciated. the error i get is:

Error:
Traceback (most recent call last): File "C:\Python\Python36-32\TUTORIALS\chatbot\chatbot-database.py", line 104, in <module> row = json.loads(row) File "C:\Python\Python36-32\lib\json\__init__.py", line 354, in loads return _default_decoder.decode(s) File "C:\Python\Python36-32\lib\json\decoder.py", line 339, in decode obj, end = self.raw_decode(s, idx=_w(s, 0).end()) File "C:\Python\Python36-32\lib\json\decoder.py", line 355, in raw_decode obj, end = self.scan_once(s, idx) json.decoder.JSONDecodeError: Unterminated string starting at: line 1 column 374 (char 373)
the code i'm trying to run is:

import sqlite3
import json
from datetime import datetime

timeframe  = '2015-01'
sql_transaction = []

connection = sqlite3.connect('{}.db'.format(timeframe))
c = connection.cursor()

def create_table():
    c.execute("""CREATE TABLE IF NOT EXISTS parent_reply
              (parent_id TEXT PRIMARY KEY, comment_id TEXT UNIQUE,
              parent TEXT, comment TEXT, subbreddit TEXT, unix INT, score INT)""")

def format_data(data):
    data = data.replace("\n", " newlinechar ").replace("\r", " returnchar ").replace('"', "'")
    return data

def find_existing_score(pid):
    try:

        sql = "SELECT score FROM parent_reply WHERE parent_id = '{}' LIMIT 1".format(pid)
        c.execute(sql)
        result = c.fetchone()
        if result != None:
            return result[0]
        else: return False
    except Exception as e:
        #print("find_parent", e)
        return False

def acceptable(data):
    if len(data.split(' ')) > 50 or len(data) < 1:
        return False
    elif len(data) > 1000:
        return False
    elif data == '[deleted]' or data == '[removed]':
        return False
    else:
        return True
    
    
def find_parent(pid):
    try:

        sql = "SELECT comment FROM parent_reply WHERE comment_id = '{}' LIMIT 1".format(pid)
        c.execute(sql)
        result = c.fetchone()
        if result != None:
            return result[0]
        else: return False
    except Exception as e:
        #print("find_parent", e)
        return False

def transaction_bldr(sql):
    global sql_transaction
    sql_transaction.append(sql)
    if len(sql_transaction) > 1000:
        c.execute('BEGIN TRANSACTION')
        for s in sql_transaction:
            try:
                c.execute(s)
            except:
                pass
        connection.commit()
        sql_transaction = []
            
    

def sql_insert_replace_comment(commentid, parentid, parent, comment, subreddit, time, score):
    try:
        sql = """UPDATE parent_reply SET parent_id = ?, comment_id = ?, parent = ?, comment = ?, subreddit = ?, time = ?, score = ? WHERE parent_id = ?;""".format(parentid, commentid, parent, comment, subreddit, time, score)
        transaction_bldr(sql)
    except Exception as e:
        print('s-UPDATE insertion', str(e))

def sql_insert_has_parent(commentid, parentid, parent, comment, subreddit, time, score):
    try:
        sql = """INSERT INTO parent_reply (parent_id, comment_id, parent, comment, subreddit, time, score)""".format(parentid, commentid, parent, comment, subreddit, time, score)
        transaction_bldr(sql)
    except Exception as e:
        print('s-PARENT insertion', str(e))

def sql_insert_no_parent(commentid, parentid, comment, subreddit, time, score):
    try:
        sql = """INSERT INTO parent_reply (parent_id, comment_id, comment, subreddit, time, score)""".format(parentid, commentid, comment, subreddit, time, score)
        transaction_bldr(sql)
    except Exception as e:
        print('s-NO_PARENT insertion', str(e))


        
    
if __name__ == "__main__":
              create_table()
              row_counter = 0
              paired_rows = 0

              with open("E:/reddit_database/{}/RC_{}".format(timeframe.split('-')[0], timeframe), buffering=1000)as f:
                  for row in f:
                      row_counter +=1
                      row = json.loads(row)
                      parent_id = row['parent_id']
                      body = format_data(row['body'])
                      created_utc = row['created_utc']
                      score = row['score']
                      subreddit = row['subreddit']
                      comment_id = row['name']
                      parent_data = find_parent(parent_id)

                      if score >= 5:
                          if acceptable(body):
                              existing_comment_score = find_existing_score(parent_id)
                              if existing_comment_score:
                                  if score > existing_comment_score:
                                      sql_insert_replace_comment(comment_id, parent_id, parent_data, body, subreddit, created_utc, score)

                                

                              else:
                                  if parent_data:
                                      sql_insert_has_parent(comment_id, parent_id, parent_data, body, subreddit, created_utc, score)
                                      paired_rows += 1
                                  else:
                                      sql_insert_no_parent(comment_id, parent_id, body, subreddit, created_utc, score)



                      if row_counter % 100000 == 0:
                          print("Total rows read: {}, Pared rows: {}, Time {}".format(row_counter, paired_rows, str(datetime.now())))
            
Reply


Possibly Related Threads…
Thread Author Replies Views Last Post
  error 1102 (42000) incorrect database name 's' Anldra12 4 1,649 Jun-08-2022, 09:00 AM
Last Post: Anldra12
Bug beginner attempting to make chatbot MonikaDreemur 1 1,079 Feb-02-2022, 10:24 PM
Last Post: BashBedlam
  Help with simple nltk Chatbot Extra 3 1,841 Jan-02-2022, 07:50 AM
Last Post: bepammoifoge
  Python Reminder MSTeams Chatbot ajaymedidi 0 1,125 Sep-16-2021, 07:13 AM
Last Post: ajaymedidi
  mySQL Database error not resolving. cybertooth 2 3,113 Aug-30-2021, 05:45 PM
Last Post: ibreeden
  Error creating database with python and form? shams 3 2,327 Aug-02-2021, 02:00 PM
Last Post: deanhystad
  Generate questions from excel to test my chatbot mcubac08 5 2,827 Sep-01-2020, 06:15 PM
Last Post: mcubac08
  Trying to pass an exe filepath to tkinter Chatbot & expecting the chatbot to run it svkroy 0 1,557 Jul-16-2020, 07:46 AM
Last Post: svkroy
  Chatbot metro17 2 9,878 Sep-21-2019, 02:05 PM
Last Post: Larz60+
  Error in Database connectivity with python3.7 srm 8 4,406 May-13-2019, 07:28 AM
Last Post: srm

Forum Jump:

User Panel Messages

Announcements
Announcement #1 8/1/2020
Announcement #2 8/2/2020
Announcement #3 8/6/2020