import scrapy
from scrapy import Field


class SubjectItem(scrapy.Item):
    # define the fields for your item here like:
    id = Field()
    _id = Field()
    name = Field()
    image = Field()
    subject_type = Field()
    name_cn = Field()
    tags = Field()
    info = Field()

    score = Field()

    score_details = Field()

    wishes = Field()
    done = Field()
    doings = Field()
    on_hold = Field()
    dropped = Field()

# -*- coding: utf-8 -*-
from typing import List

import pymongo
from scrapy import Request
from bgm.items import SubjectItem, RelationItem

def url_from_id(_id):
    return 'https://mirror.bgm.rin.cat/subject/{}'.format(_id)

class BgmTvSpider(scrapy.Spider):
    name = 'bgm_tv'
    allowed_domains = ['mirror.bgm.rin.cat']
    start_urls = ['https://mirror.bgm.rin.cat/subject/{}'.format(i)
                  for i in range(1, 270000)]

    def parse(self, response):
        if '出错了' not in response.text:
            subject_item = SubjectItem()

            subject_item['subject_type'] = get_subject_type(response)

            if subject_item['subject_type'] == 'Music':
                return

            subject_item['_id'] = int(response.url.split('/')[-1])
            subject_item['id'] = subject_item['_id']

            subject_item['info'] = get_info(response)
            subject_item['tags'] = get_teg_from_response(response)
            subject_item['image'] = get_image(response)
            subject_item['score'] = get_score(response)
            subject_item['score_details'] = get_score_details(response)

            title = response.xpath('//*[@id="headerSubject"]/h1/a')[0]

            subject_item['name_cn'] = title.attrib['title']
            subject_item['name'] = title.xpath('text()').extract_first()

            for edge in get_relation(response, source=subject_item['_id']):
                relation_item = RelationItem(**edge, )
                yield relation_item
                yield Request(url_from_id(relation_item['target']))
            yield subject_item

{
  "nodes": [
    { "name": "云天河", "image": "tianhe.png" },
    { "name": "韩菱纱", "image": "lingsha.png" },
    { "name": "柳梦璃", "image": "mengli.png" },
    { "name": "慕容紫英", "image": "ziying.png" }
  ],
  "edges": [
    { "source": 0, "target": 1, "relation": "挚友" },
    { "source": 0, "target": 2, "relation": "挚友" },
    { "source": 0, "target": 3, "relation": "挚友" }
  ]
}

def worker(start_job=None):
    yield_job = []
    done_id = set()
    if start_job is None:
        start_job = [
            x['_id'] for x in n_subject.find({}, {'_id': 1})
        ]
    while True:
        print('\r', len(yield_job) + len(start_job), end='')
        if yield_job:
            j = yield_job.pop()
            if j in done_id:
                continue
            for node in deal_with_node({'_id': j}):
                yield_job.append(node)
            done_id.add(j)
        elif start_job:
            j = start_job.pop()
            if j in done_id:
                continue
            for node in deal_with_node({'_id': j}):
                yield_job.append(node)
            done_id.add(j)
        else:
            print('done')
            break

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymongo

from bgm.items import SubjectItem, RelationItem
from typing import Union
from bgm.models import Subject, Relation
from twisted.enterprise import adbapi
import bgm.settings


# from playhouse.shortcuts import keyli

class MysqlPipeline(object):
    def open_spider(self, spider):
        self.dbpool = adbapi.ConnectionPool(
            "MySQLdb",
            host=bgm.settings.MYSQL_HOST,
            db=bgm.settings.MYSQL_DBNAME,
            user=bgm.settings.MYSQL_USER,
            password=bgm.settings.MYSQL_PASSWORD,
            charset='utf8mb4',
        )

    def process_item(self,
                     item: Union[SubjectItem, RelationItem],
                     spider):
        query = self.dbpool.runInteraction(self.do_insert, item)
        # 处理异常
        query.addErrback(self.handle_error, item, spider)
        return item

    def handle_error(self, failure, item, spider):
        # 处理异步插入的异常
        print(failure)

    def do_insert(self, cursor, item):
        # 会从dbpool取出cursor
        # 执行具体的插入
        if isinstance(item, SubjectItem):
            if not item['name']:
                item['name'] = item['name_cn']
            # if not item['name_cn']:
            #     item['name_cn'] = item['name']
            insert_sql = Subject.insert(
                **item
            ).on_conflict_replace().sql()
        elif isinstance(item, RelationItem):
            insert_sql = Relation.insert(
                id=f'{item["source"]}-{item["target"]}',
                **item
            ).on_conflict_replace().sql()
        else:
            return
        cursor.execute(*insert_sql)
    # 拿传进的cursor进行执行，并且自动完成commit操作