Add check_links.py utility script

check_links.py

@@ -0,0 +1,300 @@
+#!/usr/bin/env python3
+"""
+检测 awesome-openclaw-skills README.md 中链接的有效性。
+使用 HEAD 请求检测状态码，支持通过 GITHUB_TOKEN 环境变量提高 GitHub API 并发限制。
+"""
+
+import argparse
+import os
+import re
+import ssl
+import sys
+import time
+import urllib.request
+import urllib.error
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from http.client import HTTPResponse
+from typing import Optional
+from urllib.parse import urlparse
+
+
+@dataclass
+class LinkResult:
+    """链接检测结果"""
+    name: str
+    url: str
+    line_num: int
+    original_line: str
+    status_code: Optional[int]
+    error: Optional[str]
+    is_valid: bool
+
+
+def extract_links_from_readme(filepath: str) -> list[tuple[str, str, int, str]]:
+    """
+    从 README.md 中提取所有技能链接。
+    
+    返回: [(skill_name, url, line_num, original_line), ...]
+    """
+    pattern = re.compile(r'-\s+\[([^\]]+)\]\((https://github\.com/openclaw/skills/[^\)]+)\)')
+    
+    links = []
+    with open(filepath, 'r', encoding='utf-8') as f:
+        for line_num, line in enumerate(f, 1):
+            match = pattern.search(line)
+            if match:
+                name, url = match.groups()
+                links.append((name, url, line_num, line.rstrip('\n')))
+    
+    return links
+
+
+def check_link(name: str, url: str, github_token: Optional[str], timeout: int = 10) -> tuple[Optional[int], Optional[str], bool]:
+    """
+    使用 HEAD 请求检测单个链接的有效性。
+    
+    对于 GitHub 链接，使用 GITHUB_TOKEN 进行认证以提高 API 限制。
+    GitHub API 限制：
+    - 未认证：60 次/小时
+    - 认证后：5000 次/小时
+    
+    返回: (status_code, error_msg, is_valid)
+    """
+    # 构建请求
+    parsed = urlparse(url)
+    
+    # 将 github.com 链接转换为 API 调用以获取更准确的状态
+    # 例如：https://github.com/openclaw/skills/tree/main/skills/xxx/SKILL.md
+    # 转换为：https://api.github.com/repos/openclaw/skills/contents/skills/xxx/SKILL.md?ref=main
+    
+    is_github = parsed.netloc == 'github.com'
+    
+    if is_github:
+        # 解析 GitHub URL 路径
+        path_parts = parsed.path.split('/')
+        # /openclaw/skills/tree/main/skills/author/skill-name/SKILL.md
+        if len(path_parts) >= 6 and path_parts[3] == 'tree':
+            repo_owner = path_parts[1]
+            repo_name = path_parts[2]
+            branch = path_parts[4]
+            file_path = '/'.join(path_parts[5:])
+            
+            # 构建 GitHub API URL
+            api_url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/contents/{file_path}?ref={branch}"
+            check_url = api_url
+        else:
+            check_url = url
+    else:
+        check_url = url
+    
+    # 创建请求
+    req = urllib.request.Request(check_url, method='HEAD')
+    
+    # 设置请求头
+    req.add_header('User-Agent', 'awesome-openclaw-skills-link-checker/1.0')
+    
+    if is_github and github_token:
+        req.add_header('Authorization', f'token {github_token}')
+        # GitHub API 需要 Accept 头
+        req.add_header('Accept', 'application/vnd.github.v3+json')
+    
+    # 创建 SSL 上下文
+    ssl_context = ssl.create_default_context()
+    
+    try:
+        with urllib.request.urlopen(req, timeout=timeout, context=ssl_context) as response:
+            # 对于 GitHub API，HEAD 请求可能不支持，需要处理
+            if isinstance(response, HTTPResponse):
+                status_code = response.status
+            else:
+                status_code = 200
+            
+            is_valid = 200 <= status_code < 400
+            
+            return (status_code, None, is_valid)
+    
+    except urllib.error.HTTPError as e:
+        status_code = e.code
+        error_msg = None
+        
+        if status_code == 404:
+            error_msg = "Not Found"
+            is_valid = False
+        elif status_code == 403:
+            error_msg = "Forbidden (rate limited?)"
+            # 速率限制表示资源存在，只是暂时无法访问
+            is_valid = True
+        elif status_code == 429:
+            error_msg = "Too Many Requests"
+            # 速率限制表示资源存在，只是暂时无法访问
+            is_valid = True
+        else:
+            error_msg = f"HTTP {status_code}"
+            is_valid = False
+        
+        return (status_code, error_msg, is_valid)
+    
+    except urllib.error.URLError as e:
+        return (None, f"URL Error: {e.reason}", False)
+    
+    except TimeoutError:
+        return (None, "Timeout", False)
+    
+    except Exception as e:
+        return (None, f"Error: {str(e)}", False)
+
+
+def check_all_links(
+    links: list[tuple[str, str, int, str]],
+    github_token: Optional[str],
+    max_workers: int = 10,
+    rate_limit_delay: float = 0.1
+) -> list[LinkResult]:
+    """
+    并发检测所有链接。
+    
+    参数:
+        links: [(name, url, line_num, original_line), ...]
+        github_token: GitHub 个人访问令牌
+        max_workers: 最大并发数
+        rate_limit_delay: 每次请求之间的延迟（秒）
+    """
+    results = []
+    total = len(links)
+    
+    print(f"开始检测 {total} 个链接...")
+    print(f"并发数: {max_workers}")
+    print(f"GITHUB_TOKEN: {'已设置' if github_token else '未设置 (限制: 60次/小时)'}")
+    print("-" * 60)
+    
+    def check_with_delay(link_tuple):
+        name, url, line_num, original_line = link_tuple
+        status_code, error, is_valid = check_link(name, url, github_token)
+        time.sleep(rate_limit_delay)  # 添加延迟以避免触发速率限制
+        return LinkResult(
+            name=name,
+            url=url,
+            line_num=line_num,
+            original_line=original_line,
+            status_code=status_code,
+            error=error,
+            is_valid=is_valid
+        )
+    
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {
+            executor.submit(check_with_delay, link): link
+            for link in links
+        }
+        
+        completed = 0
+        for future in as_completed(futures):
+            completed += 1
+            result = future.result()
+            results.append(result)
+            
+            # 显示进度（始终打印 URL）
+            status_icon = "✓" if result.is_valid else "✗"
+            if result.is_valid:
+                print(f"[{completed}/{total}] {status_icon} {result.name}")
+                print(f"    {result.url}")
+            else:
+                error_info = result.error or f"HTTP {result.status_code}"
+                print(f"[{completed}/{total}] {status_icon} {result.name} - {error_info}")
+                print(f"    {result.url}")
+    
+    return results
+
+
+def delete_invalid_lines(readme_path: str, results: list[LinkResult]) -> int:
+    """
+    删除 README.md 中无效链接所在的行。
+    
+    返回: 删除的行数
+    """
+    # 收集需要删除的行号
+    invalid_lines = {r.line_num for r in results if not r.is_valid}
+    
+    if not invalid_lines:
+        return 0
+    
+    # 读取所有行
+    with open(readme_path, 'r', encoding='utf-8') as f:
+        lines = f.readlines()
+    
+    # 过滤掉无效行
+    new_lines = [
+        line for line_num, line in enumerate(lines, 1)
+        if line_num not in invalid_lines
+    ]
+    
+    # 写回文件
+    with open(readme_path, 'w', encoding='utf-8') as f:
+        f.writelines(new_lines)
+    
+    return len(invalid_lines)
+
+
+def main():
+    # 解析命令行参数
+    parser = argparse.ArgumentParser(description='检测 README.md 中链接的有效性')
+    parser.add_argument('--delete', action='store_true', help='删除无效链接所在的行')
+    args = parser.parse_args()
+    
+    # 获取 README.md 路径
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    readme_path = os.path.join(script_dir, "README.md")
+    
+    if not os.path.exists(readme_path):
+        print(f"错误: 找不到 README.md 文件: {readme_path}")
+        sys.exit(1)
+    
+    # 获取 GITHUB_TOKEN
+    github_token = os.environ.get("GITHUB_TOKEN")
+    
+    # 提取链接
+    print(f"正在读取 {readme_path}...")
+    links = extract_links_from_readme(readme_path)
+    print(f"找到 {len(links)} 个链接")
+    print()
+    
+    if not links:
+        print("没有找到任何链接")
+        sys.exit(0)
+    
+    # 检测链接
+    # 对于 GitHub API，如果使用 token，可以更高并发
+    # 未使用 token 时，降低并发以避免触发速率限制
+    max_workers = 20 if github_token else 5
+    rate_limit_delay = 0.05 if github_token else 0.5
+    
+    results = check_all_links(
+        links,
+        github_token,
+        max_workers=max_workers,
+        rate_limit_delay=rate_limit_delay
+    )
+    
+    # 统计结果
+    print()
+    print("=" * 60)
+    valid_count = sum(1 for r in results if r.is_valid)
+    invalid_count = len(results) - valid_count
+    print(f"检测完成: 有效 {valid_count}, 无效 {invalid_count}")
+    
+    # 如果需要，删除无效行
+    if args.delete and invalid_count > 0:
+        print()
+        print("正在删除无效链接...")
+        deleted = delete_invalid_lines(readme_path, results)
+        print(f"已删除 {deleted} 行")
+    
+    # 返回退出码
+    if invalid_count > 0:
+        sys.exit(1)
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()