AI Automation Pipeline: Scrape → Rewrite → Post Otomatis — 100 Artikel/Bulan dari Satu VPS

import feedparser
import json
from datetime import datetime

def fetch_feeds(feed_urls, max_per_feed=5):
    """Pull latest articles from RSS feeds."""
    articles = []
    
    for url in feed_urls:
        feed = feedparser.parse(url)
        print(f"Feed: {feed.feed.get('title', url)} — {len(feed.entries)} entries")
        
        for entry in feed.entries[:max_per_feed]:
            article = {
                "source_title": feed.feed.get("title", "Unknown"),
                "title": entry.get("title", ""),
                "url": entry.get("link", ""),
                "summary": entry.get("summary", ""),
                "published": entry.get("published", datetime.now().isoformat()),
                "content": entry.get("content", [{"value": ""}])[0]["value"] if entry.get("content") else entry.get("summary", "")
            }
            articles.append(article)
    
    return articles

# Sumber feeds — sesuaikan sama niche lo
FEEDS = [
    "https://news.ycombinator.com/rss",
    "https://techcrunch.com/feed/",
    "https://www.reddit.com/r/startups/.rss",
    # Tambahin 10-20 feed niche lo di sini
]

articles = fetch_feeds(FEEDS)
print(f"Total: {len(articles)} artikel dari feed")

import requests
from bs4 import BeautifulSoup
from readability import Document
import html2text

def scrape_article(url):
    """Scrape article content from URL using readability."""
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
    }
    
    resp = requests.get(url, headers=headers, timeout=30)
    resp.raise_for_status()
    
    # Extract main content with readability
    doc = Document(resp.text)
    content_html = doc.summary()
    title = doc.title()
    
    # Convert HTML to markdown
    converter = html2text.HTML2Text()
    converter.ignore_links = False
    converter.ignore_images = True
    content_md = converter.handle(content_html)
    
    return {
        "title": title,
        "content": content_md,
        "url": url
    }

# Test scrape satu artikel
print(scrape_article("https://example.com/some-article"))

pip install feedparser beautifulsoup4 readability-lxml html2text requests

import requests
import re

def rewrite_article(local_model_url, article):
    """Rewrite article using local abliterated model.
    
    Args:
        local_model_url: http://localhost:11434/api/generate
        article: dict with title, content, source_title
    
    Returns:
        dict with new_title, new_content
    """
    prompt = f"""Rewrite this article completely. Rules:

1. STRUCTURE must be different — don't follow original paragraph order
2. All sentences must be rewritten — no sentence copied verbatim
3. Add new insights, examples, or angles (at least 30% new content)
4. Keep factual information accurate
5. Target: Indonesian entrepreneurs / tech enthusiasts
6. Tone: practical, direct, no fluff
7. Length: 800-1500 words
8. Format: markdown with subheadings (## and ###)

Original Source: {article['source_title']}
Original Title: {article['title']}

Original Content:
{article['content'][:4000]}  # Limit to 4000 chars

---

Now produce the rewritten article in Indonesian. Start with a new title on the first line.
"""
    
    resp = requests.post(local_model_url, json={
        "model": "noushermes2:7b",
        "prompt": prompt,
        "stream": False,
        "options": {
            "temperature": 0.8,
            "num_predict": 4096
        }
    }, timeout=180)
    
    result = resp.json()["response"]
    
    # Extract title (first line) and content
    lines = result.strip().split("\n", 1)
    new_title = lines[0].replace("# ", "").strip()
    new_content = lines[1] if len(lines) > 1 else result
    
    return {
        "new_title": new_title,
        "new_content": new_content
    }

from nltk.corpus import wordnet

def synonym_rotate(text):
    """Replace words with synonyms for basic obfuscation."""
    words = text.split()
    new_words = []
    for word in words:
        # Skip short words and stopwords
        if len(word) < 5 or word.lower() in ["the", "and", "for", "are", "but", "not", "you", "all", "can", "had", "her", "was", "one", "our", "out", "has", "have", "been", "some", "same"]:
            new_words.append(word)
            continue
        
        # Try to find a synonym
        synsets = wordnet.synsets(word)
        if synsets:
            lemmas = [l.name().replace("_", " ") for l in synsets[0].lemmas()]
            # Pick a different lemma that's not the original word
            alternatives = [l for l in lemmas if l.lower() != word.lower()]
            if alternatives:
                new_words.append(alternatives[0])
                continue
        
        new_words.append(word)
    
    return " ".join(new_words)

import os
from datetime import datetime
import re

def slugify(title):
    """Convert title to URL-friendly slug."""
    slug = title.lower()
    slug = re.sub(r'[^a-z0-9\s-]', '', slug)
    slug = re.sub(r'[\s-]+', '-', slug)
    return slug[:80].strip('-')

def create_hugo_post(new_title, new_content, tags=None):
    """Wrap content in Hugo frontmatter + markdown."""
    today = datetime.now().strftime("%Y-%m-%d")
    slug = slugify(new_title)
    tags_str = str(tags or ["ai", "automation", "tutorial"])
    
    # Generate excerpt (first 160 chars of content)
    excerpt = new_content[:160].replace("\n", " ").strip()
    
    frontmatter = f"""---
title: "{new_title}"
date: {today}
draft: false
description: "{excerpt}..."
tags: {tags_str}
---

"""
    
    # Remove any title line from content (already in frontmatter)
    content_lines = new_content.split("\n")
    if content_lines[0].startswith("#"):
        content_lines = content_lines[1:]
    body = "\n".join(content_lines).strip()
    
    return frontmatter + body, slug

import subprocess
import os

def commit_and_deploy(posts_dir, slug, markdown_content):
    """Write file, git commit, push to trigger Hugo deploy."""
    # Write markdown file
    filepath = os.path.join(posts_dir, f"{slug}.md")
    with open(filepath, "w", encoding="utf-8") as f:
        f.write(markdown_content)
    
    print(f"Written: {filepath}")
    
    # Git commit + push
    subprocess.run(["git", "-C", posts_dir, "add", f"{slug}.md"], check=True)
    subprocess.run(["git", "-C", posts_dir, "commit", "-m", f"Auto-post: {slug}"], check=True)
    subprocess.run(["git", "-C", posts_dir, "push"], check=True)
    
    print(f"Committed and pushed: {slug}")

#!/usr/bin/env python3
"""
AutoBlog Pipeline v2 — Scrape → Rewrite → Post
Jalanin dari cron: */360 * * * * /home/ubuntu/auto_blog.py >> /var/log/auto_blog.log 2>&1
"""

import feedparser
import requests
import os
import subprocess
import json
import re
import random
from datetime import datetime

# Konfigurasi
OLLAMA_URL = "http://localhost:11434/api/generate"
MODEL_NAME = "noushermes2:7b"
POSTS_DIR = "/home/ubuntu/peak/blog/content/posts"
MAX_ARTICLES_PER_RUN = 5  # Biar gak banjir

# Sumber feeds — ganti sama niche lo
FEEDS = [
    "https://news.ycombinator.com/rss",
    "https://techcrunch.com/feed/",
    "https://www.reddit.com/r/entrepreneur/.rss",
]

def fetch_feeds():
    """Get latest articles from all feeds."""
    articles = []
    for url in FEEDS:
        feed = feedparser.parse(url)
        for entry in feed.entries[:3]:
            articles.append({
                "source_title": feed.feed.get("title", "Unknown"),
                "title": entry.get("title", ""),
                "content": entry.get("content", [{"value": entry.get("summary", "")}])[0]["value"] if entry.get("content") else entry.get("summary", ""),
            })
    return articles

def rewrite_article(article):
    """Rewrite using local AI model."""
    # Random niche angle biar artikel gak sama
    angles = [
        "with a focus on practical steps for Indonesian entrepreneurs",
        "with cost-benefit analysis for small business owners",
        "comparing with alternatives available in Southeast Asia",
        "with a step-by-step implementation guide",
    ]
    angle = random.choice(angles)
    
    prompt = f"""Rewrite this article completely {angle}.

Rules:
- Different structure, different sentences, 30% new insights
- Target: Indonesian entrepreneurs (practical, langsung bisa dipake)
- Length: 800-1200 words
- Format: markdown with ## subheadings
- NO sentence copied verbatim from original

Original Title: {article['title']}
Original:
{article['content'][:3000]}

Rewritten article:
"""
    
    resp = requests.post(OLLAMA_URL, json={
        "model": MODEL_NAME,
        "prompt": prompt,
        "stream": False,
        "options": {"temperature": 0.8, "num_predict": 4096}
    }, timeout=180)
    
    return resp.json()["response"]

def slugify(title):
    slug = title.lower()[:80]
    slug = re.sub(r'[^a-z0-9\s-]', '', slug)
    return re.sub(r'[\s-]+', '-', slug).strip('-')

def create_post(content, source_url):
    """Generate Hugo frontmatter + save file."""
    lines = content.strip().split("\n", 1)
    title = lines[0].replace("# ", "").strip()
    body = lines[1] if len(lines) > 1 else content
    slug = slugify(title)
    today = datetime.now().strftime("%Y-%m-%d")
    
    frontmatter = f"""---
title: "{title}"
date: {today}
draft: false
description: "{body[:160].replace(chr(10), ' ').strip()}..."
tags: ["ai", "automation", "tutorial"]
---

> *Artikel ini adalah hasil rewrite dari {source_url} — dengan restrukturisasi total dan nilai tambah.*

"""
    
    filepath = os.path.join(POSTS_DIR, f"{slug}.md")
    with open(filepath, "w", encoding="utf-8") as f:
        f.write(frontmatter + body)
    
    return filepath, slug

def deploy():
    """Git commit + push."""
    subprocess.run(["git", "-C", POSTS_DIR, "add", "-A"], check=False)
    subprocess.run(["git", "-C", POSTS_DIR, "commit", "-m", f"Auto-blog: {datetime.now().isoformat()}"], check=False)
    subprocess.run(["git", "-C", POSTS_DIR, "push"], check=False)

# === MAIN ===
if __name__ == "__main__":
    print(f"[{datetime.now().isoformat()}] Starting auto-blog pipeline...")
    
    # 1. Fetch
    articles = fetch_feeds()
    print(f"Fetched {len(articles)} articles from feeds")
    
    # 2. Rewrite & post
    posted = 0
    for article in articles[:MAX_ARTICLES_PER_RUN]:
        try:
            print(f"Rewriting: {article['title'][:60]}...")
            rewritten = rewrite_article(article)
            filepath, slug = create_post(rewritten, article.get("url", ""))
            print(f"  → Saved: {slug}")
            posted += 1
        except Exception as e:
            print(f"  ✗ Error: {e}")
    
    # 3. Deploy
    if posted > 0:
        deploy()
        print(f"Deployed {posted} new articles")
    
    print(f"[{datetime.now().isoformat()}] Pipeline complete")

# Buka crontab
crontab -e

# Tambah baris ini: jalan tiap 6 jam
0 */6 * * * /usr/bin/python3 /home/ubuntu/auto_blog.py >> /var/log/auto_blog.log 2>&1

# Test kalo cron beres jalan
tail -f /var/log/auto_blog.log

def seo_optimize(title, content, keywords):
    """Optimize content for search engines."""
    prompt = f"""Improve this article for SEO. Requirements:
- Include primary keyword '{keywords[0]}' in H2 and first 100 words
- Include 3-5 secondary keywords naturally throughout
- Add internal linking suggestions [like this](https://yourblog.com/related-post)
- Optimize meta description (max 160 chars)
- Add FAQ schema at the bottom
- Ensure readability score >60 (short sentences, subheadings, bullet points)

Title: {title}
Content: {content[:3000]}
"""
    # Send to Ollama
    resp = requests.post(OLLAMA_URL, json={
        "model": MODEL_NAME,
        "prompt": prompt,
        "stream": False,
        "options": {"temperature": 0.3, "num_predict": 4096}
    })
    return resp.json()["response"]

# Install DALL-E / Stable Diffusion CLI
pip install openai

# Generate featured image per artikel
python3 -c "
import openai
openai.Image.create(
  prompt='Minimalist tech blog header: automated pipeline data flow',
  n=1,
  size='1024x1024'
)
"

pip install copyscape

# Atau pake API Copyscape buat cek tiap artikel sebelum post
curl "https://www.copyscape.com/api/" \
  -d "u=https://yourblog.com/post-slug&key=YOUR_KEY&o=xml"