Nodejs解析网站网址内容并获取标题图标

/**
 * 异步获取指定URL的内容
 * 该函数首先发送一个HTTP HEAD请求，以检查URL的内容类型和大小
 * 如果内容类型为HTML且大小在允许范围内，则进一步发送GET请求获取实际内容
 *
 * @param url 目标URL地址
 * @returns Promise对象，解析后返回URL的内容，如果发生错误则拒绝Promise
 */
export async function fetchUrlContent(url: string) {
  return axios
    .head(url, {
      validateStatus: () => true,
      maxContentLength: configs.FETCH_URL_INFO.MAX_RESPONSE_SIZE,
      headers: {
        'Content-Type': 'charset:utf-8',
        Accept: 'application/json, text/plain, */*',
        'accept-encoding': 'gzip, deflate, br'
      },
      timeout: configs.FETCH_URL_INFO.TIMEOUT
    })
    .then((res) => {
      // 检查内容大小是否超出限制
      if (res?.headers?.['content-length'] && parseInt(res?.headers['content-length']) > configs.FETCH_URL_INFO.MAX_RESPONSE_SIZE) {
        logger.log('[url] 限制：', url, res?.headers['content-length'], res?.headers['content-type']);
        return Promise.reject(new CustomError('URL_CONTENT_ERROR', '不支持该url内容解析'));
      }
      // 检查内容类型是否为HTML
      if (res?.headers['content-type']?.includes('text/html')) {
        return axios
          .get(url, { headers: { accept: 'text/html', 'Content-Type': 'text/html;charset:utf-8', 'User-Agent': configs.FETCH_URL_INFO.USER_AGENT }, timeout: configs.FETCH_URL_INFO.TIMEOUT })
          .then((res) => {
            if (res) {
              logger.log('[url] 爬取成功 axios', url);
              // 再次检查内容大小是否超出限制
              if (res.data?.length > configs.FETCH_URL_INFO.MAX_RESPONSE_SIZE) {
                logger.log('[url] buffer大小: ', url, res.data?.length);
                return Promise.reject(new CustomError('URL_CONTENT_ERROR', '内容过大解析失败'));
              }
              return res.data;
            }
            return Promise.reject(res);
          })
          .catch((e) => {
            logger.error('[url] fetch get', url, e.message);
            throw new CustomError('URL_GET_FETCH_ERROR', '不支持该url内容解析');
          });
      }
      return Promise.reject(new CustomError('URL_UNVALID_ERROR', '不支持该url内容解析'));
    })
    .catch((e) => {
      logger.error('[url] fetch head', url, e.message);
      throw new CustomError('URL_HEAD_FETCH_ERROR', '不支持该url内容解析');
    });
}

/**
 * 解析URL内容
 * @param url 页面URL
 * @param html 页面HTML内容
 * @returns 返回包含URL、图标、简介和标题的对象
 */
export async function parseUrlContent(url: string, html: string): Promise<{ url: string; icon: string; intro: string; title: string }> {
  const $ = load(html);

  let title = '';
  let intro = '';
  let icon = '';

  // 获取标题节点
  const titleEl = $('title');
  if (titleEl?.text()) {
    title = titleEl?.text();
  }

  // 获取icon
  const linkEl = $('link');
  const links: string[] = [];
  if (linkEl) {
    linkEl.each((_i, el) => {
      const rel = $(el).attr('rel');
      const href = $(el).attr('href');
      if (rel?.includes('icon') && href) {
        links.push(href);
      }
    });
  }
  logger.log('[url] 获取icon', links);
  if (links.length) {
    icon = resolveUrl(url, links[0]);
  }

  /**
   * 获取meta属性
   * @param metaElement
   * @param name
   * @returns
   */
  const getPropertyContent = (Element, name: string) => {
    const propertyName = $(Element)?.attr('property') || $(Element)?.attr('name');
    return propertyName === name ? $(Element)?.attr('content') || '' : '';
  };

  // 获取详情
  const metas = $('meta');
  for (const meta of metas) {
    if (title && intro) {
      break;
    }
    // 如果没有标题
    if (!title) {
      const titleoAttr = ['og:title', 'twitter:title'];
      for (const attr of titleoAttr) {
        const text = getPropertyContent(meta, attr);
        if (text) {
          title = text;
          break;
        }
      }
    }

    // 简介
    if (!intro) {
      const introAttr = ['description', 'og:description', 'twitter:description'];
      for (const attr of introAttr) {
        const description = getPropertyContent(meta, attr);
        if (description) {
          intro = description;
          break;
        }
      }
    }

    // icon
    if (!icon) {
      const imageAttr = ['og:image', 'twitter:image'];
      for (const attr of imageAttr) {
        const image = getPropertyContent(meta, attr);
        if (image) {
          intro = resolveUrl(url, image);
          break;
        }
      }
    }
  }
  // 没有简介提取全部
  if (!intro) {
    const body = $('body').html();
    intro = body ? htmlStringReplace(body, configs.FETCH_URL_INFO.MAX_INTRO_LENGTH) : '';
  }
  logger.log('[url] 爬取结果', { url, title, intro, icon });
  return {
    url,
    title: title?.trim() || '',
    intro: intro?.trim() || '',
    icon
  };
}

// js演示
var axios = require('axios');
var data = JSON.stringify({
  url: 'https://xygeng.cn/post/200'
});

// 注意只支持post请求
var config = {
  method: 'post',
  url: 'https://api.xygeng.cn/openapi/url/info',
  headers: {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36',
    'Content-Type': 'application/json',
    Accept: '*/*',
    Connection: 'keep-alive'
  },
  data: data
};

axios(config)
  .then(function (response) {
    console.log(JSON.stringify(response.data));
  })
  .catch(function (error) {
    console.log(error);
  });