Как увеличить охват фраз Wildberries и ускорить бота

{
  "settings": {
    "analysis": {
      "analyzer": {
        "russian_shingle": {
          "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "lowercase",
            "russian_stop",
            "snowball_russian",
            "shingle"
          ]
        },
        "russian_edge_ngram": {
          "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "lowercase",
            "russian_stop",
            "snowball_russian",
            "edge_ngram"
          ]
        }
      },
      "filter": {
        "russian_stop": {
          "type": "stop",
          "stopwords": "_russian_"
        },
        "edge_ngram": {
          "type": "edge_ngram",
          "min_gram": 2,
          "max_gram": 5
        },
        "shingle": {
          "type": "shingle",
          "min_shingle_size": 2,
          "max_shingle_size": 3,
          "output_unigrams": false
        }
      }
    }
  }
}

// Оптимальные параметры для запросов
const serpParams = {
  lang: 'ru',
  curr: 'rub',
  dest: '-1257786', // Москва
  spp: 30, // количество товаров на странице
  local_present: '1'
};

// Пример запроса
const searchUrl = `https://search.wildberries.ru/exactmatch/catalog.aspx?query=${encodeURIComponent(query)}&page=1&${new URLSearchParams(serpParams)}`;

class WildberriesParser {
  async parseSerp(url, nmId) {
    try {
      const response = await this.fetchWithRetry(url);
      const v18Data = this.tryParseV18(response);
      if (v18Data) return v18Data;
      
      const v4Data = this.tryParseV4(response);
      return v4Data;
    } catch (error) {
      this.logError('SERP parsing failed', error);
      return null;
    }
  }
}

async function checkAdPresence(nmId, query) {
  const adsUrl = `https://catalog-ads.wildberries.ru/api/v5/search?keyword=${encodeURIComponent(query)}&limit=50`;
  
  try {
    const response = await axios.get(adsUrl);
    const adItems = response.data.cards || [];
    
    const isAdvertised = adItems.some(item => item.nmId === nmId);
    const cpm = this.extractCPM(response.data);
    
    return {
      isAdvertised,
      cpm: cpm || null,
      confidence: this.calculateAdConfidence(response.data)
    };
  } catch (error) {
    return { isAdvertised: false, cpm: null, error: error.message };
  }
}

class RussianMorphology {
  constructor() {
    // Для быстрой токенизации в памяти
    this.tokenizer = new WordTokenizer();
    // Для точной нормализации - вызов внешнего сервиса
    this.pymorphy2Service = 'https://morphos.io/analyze';
  }
  
  async processQuery(query) {
    // Быстрый путь для мгновенных ответов
    const fastTokens = this.tokenizer.tokenize(query);
    const ngrams = this.generateNgrams(fastTokens, 2, 3);
    
    // Точный путь для индексации
    const normalized = await this.normalizeWithPymorphy2(query);
    
    return {
      fast: ngrams,
      accurate: normalized,
      combined: this.combineResults(ngrams, normalized)
    };
  }
}

const bottleneckConfig = {
  minTimeout: 1000, // 1 секунда между запросами
  maxTimeout: 5000, // 5 секунд при ошибках
  randomisationFactor: 0.3,
  retryAttempts: 3
};

// Пример использования
const limiter = new Bottleneck(bottleneckConfig);
const throttledFetch = limiter.wrap(axios.get);

const proxyList = [
  'http://proxy1.example.com:8080',
  'http://proxy2.example.com:8080',
  // ...
];

class SmartProxyRotator {
  constructor() {
    this.proxies = proxyList;
    this.currentIndex = 0;
  }
  
  getNextProxy() {
    const proxy = this.proxies[this.currentIndex];
    this.currentIndex = (this.currentIndex + 1) % this.proxies.length;
    return proxy;
  }
  
  async makeRequest(url, options = {}) {
    const proxy = this.getNextProxy();
    const axiosConfig = {
      ...options,
      proxy: {
        host: proxy,
        port: 8080
      },
      timeout: 10000
    };
    
    return axios.get(url, axiosConfig);
  }
}

import Queue from 'bullmq';
import { OpenSearchClient } from './opensearch';

class WildberriesBotArchitecture {
  constructor() {
    // Очереди с приоритетом
    this.instantQueue = new Queue('instant', {
      connection: redisConfig,
      defaultJobOptions: {
        removeOnComplete: 100,
        removeOnFail: 50
      }
    });
    
    this.backgroundQueue = new Queue('background', {
      connection: redisConfig,
      defaultJobOptions: {
        priority: 1,
        delay: 5000
      }
    });
    
    this.searchIndex = new OpenSearchClient();
  }
  
  async handleUserRequest(nmId) {
    // 1. Проверяем мгновенный ответ из кэша/индекса
    const instantResult = await this.getInstantResponse(nmId);
    if (instantResult) {
      return instantResult;
    }
    
    // 2. Ставим задачу в очередь с высоким приоритетом
    const job = await this.instantQueue.add('process', {
      nmId,
      userId: context.userId
    }, {
      priority: 10,
      attempts: 3,
      backoff: {
        type: 'exponential',
        delay: 2000
      }
    });
    
    // 3. Возвращаем немедленный ответ с ожиданием
    return {
      status: 'processing',
      estimatedTime: 3000,
      data: await this.getPartialResults(nmId)
    };
  }
  
  async backgroundProcessing() {
    // Фоновая обработка без приоритета
    await this.backgroundQueue.process('deepAnalysis', async (job) => {
      const { nmId } = job.data;
      await this.performDeepAnalysis(nmId);
    });
  }
}

const cacheStructure = {
  suggest: {
    ttl: 86400, // 24 часа
    prefix: 'wb:suggest:'
  },
  serp: {
    ttl: 3600, // 1 час
    prefix: 'wb:serp:'
  },
  card: {
    ttl: 172800, // 2 дня
    prefix: 'wb:card:'
  },
  ads: {
    ttl: 1800, // 30 минут
    prefix: 'wb:ads:'
  }
};

class CacheManager {
  async getOrSet(key, fetchFunction, ttl) {
    const cached = await redis.get(key);
    if (cached) return JSON.parse(cached);
    
    const data = await fetchFunction();
    await redis.setex(key, ttl, JSON.stringify(data));
    return data;
  }
}

{
  "mappings": {
    "properties": {
      "nmId": { "type": "keyword" },
      "phrases": {
        "type": "text",
        "analyzer": "russian_shingle",
        "fields": {
          "keyword": { "type": "keyword" },
          "edge": { "type": "text", "analyzer": "russian_edge_ngram" }
        }
      },
      "positions": {
        "type": "object",
        "properties": {
          "organic": { "type": "integer" },
          "advertised": { "type": "boolean" },
          "lastUpdated": { "type": "date" }
        }
      },
      "scores": {
        "type": "object",
        "properties": {
          "frequency": { "type": "float" },
          "consistency": { "type": "float" },
          "trend": { "type": "float" }
        }
      }
    }
  }
}