Sharing ideas on processing tens of millions of data in a single MySQL table

select a,b,c from db.tb limit 10000 offset 9000000

select a,b,c from db.tb where id=9999999;
update db.tb set a=x where id=9999999;

def mission_handler(all_missions, worker_mission_size):
    """
    The task list is calculated based on the total number of tasks and the number of tasks of each worker. The task list elements are (task start id, task end id).
    Example: The total number of tasks is 100, and the number of tasks for each worker is 40. Then the task list is: [(1, 40), (41, 80), (81, 100)]
    :param all_missions: total number of missions :param worker_mission_size: maximum number of missions for each worker :return: [(start_id, end_id), (start_id, end_id), ...]
    """
    worker_mission_ids = []
    current_id = 0
    while current_id <= all_missions:
        start_id = all_missions if current_id + 1 >= all_missions else current_id + 1
        end_id = all_missions if current_id + worker_mission_size >= all_missions else current_id + worker_mission_size
        if start_id == end_id:
            if worker_mission_ids[-1][1] == start_id:
                break
        worker_mission_ids.append((start_id, end_id))
        current_id += worker_mission_size

    return worker_mission_ids

>>> mission_handler(100, 40)
[(1, 40), (41, 80), (81, 100)]

from concurrent.futures import ProcessPoolExecutor


def main():
    # Maximum value of the auto-increment id max_id = 30000000
    # Data volume processed by a single worker worker_mission_size = 1000000
    # Use multiple processes to process missions = mission_handler(max_id, worker_mission_size)
    workers = []
    executor = ProcessPoolExecutor()
    for idx, mission in enumerate(missions):
        start_id, end_id = mission
        workers.append(executor.submit(data_handler, start_id, end_id, idx))


def data_handler(start_id, end_id, worker_id):
    pass

# Use another table to record the processing status insert into db.tb_handle_status(row_id, success) values ​​(999, 0);

def data_handler(start_id, end_id, worker_id):
    #Data connection conn, cursor = mysql()
    current_id = start_id
        try:
            while current_id <= end_id:
                try:
                    # TODO data processing code pass

                except Exception as e:
                    # TODO record processing results# data moves to the next current_id += 1
                    continue
                else:
                    # No exception, continue to process the next data current_id += 1
        except Exception as e:
            return 'worker_id({}): result({})'.format(worker_id, False)
        finally:
            # Database resource release cursor.close()
            conn.close()

        return 'worker_id({}): result({})'.format(worker_id, True)

sql = """update db.tb set a=%s, b=%s where id=%s"""
values ​​= [
            ('a_value', 'b_value', 9999),
            ('a_value', 'b_value', 9998),
            ...
         ]
# Batch submission to reduce network io and lock acquisition frequency cursor.executemany(sql, values)