import nltk
from nltk.tokenize import word_tokenize, sent_tokenize
nltk.download('punkt')

text = "Natural Language Processing is fascinating! It powers ChatGPT and many AI applications."

# 단어 토큰화
word_tokens = word_tokenize(text)
print("단어 토큰:", word_tokens)
# 출력: ['Natural', 'Language', 'Processing', 'is', 'fascinating', '!', ...]

# 문장 토큰화
sent_tokens = sent_tokenize(text)
print("문장 토큰:", sent_tokens)
# 출력: ['Natural Language Processing is fascinating!', 'It powers ChatGPT...']

text = "Hello NLP"
char_tokens = list(text)
print("문자 토큰:", char_tokens)
# 출력: ['H', 'e', 'l', 'l', 'o', ' ', 'N', 'L', 'P']

from tokenizers import ByteLevelBPETokenizer

# BPE 토크나이저 훈련
tokenizer = ByteLevelBPETokenizer()
tokenizer.train(
    files=["corpus.txt"],
    vocab_size=5000,
    min_frequency=2,
    special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>"]
)

# 인코딩
encoding = tokenizer.encode("Natural Language Processing")
print("토큰:", encoding.tokens)
print("ID:", encoding.ids)

import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
nltk.download('stopwords')

text = "This is a sample sentence showing off the stop words filtration."
stop_words = set(stopwords.words('english'))

word_tokens = word_tokenize(text)
filtered_sentence = [w for w in word_tokens if not w.lower() in stop_words]

print("원본:", word_tokens)
print("필터링 후:", filtered_sentence)
# 출력: ['sample', 'sentence', 'showing', 'stop', 'words', 'filtration', '.']

from nltk.stem import PorterStemmer, LancasterStemmer

ps = PorterStemmer()
ls = LancasterStemmer()

words = ["running", "runs", "ran", "runner", "easily", "fairly"]
for word in words:
    print(f"{word:15} -> Porter: {ps.stem(word):15} Lancaster: {ls.stem(word)}")

import nltk
from nltk.stem import WordNetLemmatizer
nltk.download('wordnet')

lemmatizer = WordNetLemmatizer()

# 품사를 지정하면 더 정확한 결과
print(lemmatizer.lemmatize("running", pos='v'))  # run
print(lemmatizer.lemmatize("better", pos='a'))    # good
print(lemmatizer.lemmatize("dogs"))               # dog
print(lemmatizer.lemmatize("went", pos='v'))      # go

import re

def clean_text(text):
    """텍스트 정제 함수"""
    # HTML 태그 제거
    text = re.sub(r'<[^>]+>', '', text)

    # URL 제거
    text = re.sub(r'http\S+|www\S+|https\S+', '', text, flags=re.MULTILINE)

    # 이메일 제거
    text = re.sub(r'\S+@\S+', '', text)

    # 특수문자 제거 (알파벳, 숫자, 공백만 허용)
    text = re.sub(r'[^a-zA-Z0-9\s]', '', text)

    # 여러 공백을 하나로
    text = re.sub(r'\s+', ' ', text)

    # 앞뒤 공백 제거
    text = text.strip().lower()

    return text

sample_text = """
Check out my website at https://example.com!
Email me at user@example.com for <b>more info</b>.
It's   really   cool!!!
"""

cleaned = clean_text(sample_text)
print(cleaned)
# 출력: "check out my website at email me at for more info its really cool"

import spacy

# 영어 모델 로드
nlp = spacy.load("en_core_web_sm")

text = "Apple is looking at buying U.K. startup for $1 billion"
doc = nlp(text)

print("=== 토큰 정보 ===")
for token in doc:
    print(f"{token.text:15} | POS: {token.pos_:10} | Lemma: {token.lemma_:15} | Stop: {token.is_stop}")

print("\n=== 개체명 인식 ===")
for ent in doc.ents:
    print(f"{ent.text:20} -> {ent.label_} ({spacy.explain(ent.label_)})")

print("\n=== 의존성 파싱 ===")
for token in doc:
    print(f"{token.text:15} -> {token.dep_:15} (head: {token.head.text})")

from konlpy.tag import Okt, Mecab, Komoran, Kkma

# Okt (Open Korean Text) 사용 예제
okt = Okt()

text = "자연어 처리는 인공지능의 핵심 분야입니다."

print("형태소 분석:", okt.morphs(text))
print("품사 태깅:", okt.pos(text))
print("명사 추출:", okt.nouns(text))
print("어절 분리:", okt.phrases(text))

# 출력 예시:
# 형태소 분석: ['자연어', '처리', '는', '인공지능', '의', '핵심', '분야', '입니다', '.']
# 품사 태깅: [('자연어', 'Noun'), ('처리', 'Noun'), ('는', 'Josa'), ...]
# 명사 추출: ['자연어', '처리', '인공지능', '핵심', '분야']

# Mecab - 가장 빠르고 정확한 형태소 분석기
try:
    mecab = Mecab()
    print("\nMecab 분석:")
    print(mecab.pos(text))
except Exception:
    print("Mecab이 설치되지 않았습니다.")

# 한국어 불용어 처리
korean_stopwords = ['은', '는', '이', '가', '을', '를', '의', '에', '에서', '으로', '와', '과', '도']

def korean_tokenize(text):
    okt = Okt()
    tokens = okt.morphs(text, stem=True)  # 어간 추출 포함
    tokens = [t for t in tokens if t not in korean_stopwords]
    tokens = [t for t in tokens if len(t) > 1]  # 1글자 제거
    return tokens

print("\n한국어 토큰화:", korean_tokenize(text))

from sklearn.feature_extraction.text import CountVectorizer
import numpy as np

corpus = [
    "I love natural language processing",
    "Natural language processing is amazing",
    "I love machine learning too",
    "Deep learning is part of machine learning"
]

# BoW 벡터라이저
vectorizer = CountVectorizer()
X = vectorizer.fit_transform(corpus)

print("어휘:", vectorizer.get_feature_names_out())
print("\nBoW 행렬:")
print(X.toarray())
print("\n형태:", X.shape)  # (4 문서, n 단어)

# 특정 문서 확인
doc_0 = dict(zip(vectorizer.get_feature_names_out(), X.toarray()[0]))
print("\n문서 0의 단어 빈도:")
for word, count in sorted(doc_0.items()):
    if count > 0:
        print(f"  {word}: {count}")

from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd

corpus = [
    "the cat sat on the mat",
    "the cat sat on the hat",
    "the dog sat on the log",
    "the cat wore the hat",
]

# TF-IDF 벡터라이저
tfidf = TfidfVectorizer(smooth_idf=True, norm='l2')
X = tfidf.fit_transform(corpus)

# 결과를 데이터프레임으로 표시
df = pd.DataFrame(
    X.toarray(),
    columns=tfidf.get_feature_names_out(),
    index=[f"Doc {i}" for i in range(len(corpus))]
)
print(df.round(3))

# TF-IDF 수동 계산 예시
def compute_tfidf(corpus):
    from math import log

    # 어휘 구축
    vocab = set()
    for doc in corpus:
        vocab.update(doc.split())
    vocab = sorted(vocab)

    # TF 계산
    def tf(word, doc):
        words = doc.split()
        return words.count(word) / len(words)

    # IDF 계산
    def idf(word, corpus):
        n_docs_with_word = sum(1 for doc in corpus if word in doc.split())
        return log((1 + len(corpus)) / (1 + n_docs_with_word)) + 1

    # TF-IDF 행렬 계산
    tfidf_matrix = []
    for doc in corpus:
        row = [tf(word, doc) * idf(word, corpus) for word in vocab]
        tfidf_matrix.append(row)

    return tfidf_matrix, vocab

matrix, vocab = compute_tfidf(corpus)
print("\n수동 계산 TF-IDF:")
df_manual = pd.DataFrame(matrix, columns=vocab)
print(df_manual.round(3))

from nltk.util import ngrams
from nltk.tokenize import word_tokenize
from collections import Counter

text = "I love natural language processing and machine learning"
tokens = word_tokenize(text.lower())

# Unigram (1-gram)
unigrams = list(ngrams(tokens, 1))
# Bigram (2-gram)
bigrams = list(ngrams(tokens, 2))
# Trigram (3-gram)
trigrams = list(ngrams(tokens, 3))

print("Unigrams:", unigrams[:5])
print("Bigrams:", bigrams[:5])
print("Trigrams:", trigrams[:5])

# N-gram 빈도 계산
def get_ngram_freq(text, n):
    tokens = word_tokenize(text.lower())
    n_grams = list(ngrams(tokens, n))
    return Counter(n_grams)

# 더 큰 코퍼스에서의 응용
large_corpus = """
Machine learning is a subset of artificial intelligence.
Artificial intelligence is transforming many industries.
Natural language processing is a part of machine learning.
Deep learning has revolutionized natural language processing.
"""

bigram_freq = get_ngram_freq(large_corpus, 2)
print("\n가장 빈번한 Bigrams:")
for bigram, count in bigram_freq.most_common(10):
    print(f"  {' '.join(bigram)}: {count}")

import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
from collections import Counter

class Word2VecSkipGram(nn.Module):
    """Skip-gram Word2Vec 구현"""

    def __init__(self, vocab_size, embedding_dim):
        super().__init__()
        self.center_embedding = nn.Embedding(vocab_size, embedding_dim)
        self.context_embedding = nn.Embedding(vocab_size, embedding_dim)

        # 초기화
        self.center_embedding.weight.data.uniform_(-0.5/embedding_dim, 0.5/embedding_dim)
        self.context_embedding.weight.data.uniform_(-0.5/embedding_dim, 0.5/embedding_dim)

    def forward(self, center, context, negative):
        """
        center: (batch_size,) 중심 단어 인덱스
        context: (batch_size,) 실제 문맥 단어 인덱스
        negative: (batch_size, neg_samples) 네거티브 샘플 인덱스
        """
        # 중심 단어 임베딩
        center_emb = self.center_embedding(center)  # (batch, dim)

        # 양성 샘플 점수
        context_emb = self.context_embedding(context)  # (batch, dim)
        pos_score = torch.sum(center_emb * context_emb, dim=1)  # (batch,)
        pos_loss = -torch.log(torch.sigmoid(pos_score) + 1e-10)

        # 음성 샘플 점수
        neg_emb = self.context_embedding(negative)  # (batch, neg_samples, dim)
        center_emb_expanded = center_emb.unsqueeze(1)  # (batch, 1, dim)
        neg_score = torch.bmm(neg_emb, center_emb_expanded.transpose(1, 2)).squeeze(2)  # (batch, neg_samples)
        neg_loss = -torch.sum(torch.log(torch.sigmoid(-neg_score) + 1e-10), dim=1)

        return (pos_loss + neg_loss).mean()

class Word2VecCBOW(nn.Module):
    """CBOW Word2Vec 구현"""

    def __init__(self, vocab_size, embedding_dim):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        self.linear = nn.Linear(embedding_dim, vocab_size)

    def forward(self, context):
        """
        context: (batch_size, context_window_size) 문맥 단어 인덱스
        """
        # 문맥 단어 임베딩의 평균
        embedded = self.embedding(context)  # (batch, window, dim)
        mean_embedded = embedded.mean(dim=1)  # (batch, dim)
        output = self.linear(mean_embedded)  # (batch, vocab_size)
        return output

# 데이터 준비 및 훈련 예시
def prepare_data(sentences, window_size=2):
    """훈련 데이터 준비"""
    word_counts = Counter()
    for sentence in sentences:
        word_counts.update(sentence.split())

    vocab = ['<UNK>'] + [word for word, _ in word_counts.most_common()]
    word2idx = {word: idx for idx, word in enumerate(vocab)}
    idx2word = {idx: word for word, idx in word2idx.items()}

    training_data = []
    for sentence in sentences:
        words = sentence.split()
        for i, center_word in enumerate(words):
            center_idx = word2idx.get(center_word, 0)
            for j in range(max(0, i-window_size), min(len(words), i+window_size+1)):
                if i != j:
                    context_idx = word2idx.get(words[j], 0)
                    training_data.append((center_idx, context_idx))

    return training_data, word2idx, idx2word, vocab

# Gensim을 사용한 Word2Vec (실용적인 방법)
from gensim.models import Word2Vec

sentences = [
    "I love natural language processing".split(),
    "natural language processing is a field of AI".split(),
    "machine learning is part of artificial intelligence".split(),
    "deep learning models process natural language".split(),
    "word embeddings represent words as vectors".split(),
    "Word2Vec learns word representations".split(),
    "semantic similarity between words is captured by embeddings".split(),
]

# 모델 훈련
model = Word2Vec(
    sentences=sentences,
    vector_size=100,   # 임베딩 차원
    window=5,          # 윈도우 크기
    min_count=1,       # 최소 빈도
    workers=4,         # 병렬 처리 수
    epochs=100,        # 훈련 에폭
    sg=1,              # 1=Skip-gram, 0=CBOW
    negative=5         # 네거티브 샘플링
)

# 단어 벡터 확인
print("'language' 벡터 (처음 5차원):", model.wv['language'][:5])

# 유사 단어 찾기
similar_words = model.wv.most_similar('language', topn=5)
print("\n'language'와 유사한 단어:")
for word, similarity in similar_words:
    print(f"  {word}: {similarity:.4f}")

# 단어 유추 (King - Man + Woman = Queen 유형)
# 유추: language - natural + artificial = ?
result = model.wv.most_similar(
    positive=['artificial', 'language'],
    negative=['natural'],
    topn=3
)
print("\n단어 유추 결과:")
for word, sim in result:
    print(f"  {word}: {sim:.4f}")

# 두 단어의 유사도
print("\n'language'와 'processing' 유사도:", model.wv.similarity('language', 'processing'))

from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
import numpy as np

def visualize_embeddings(model, words=None):
    """Word2Vec 임베딩 t-SNE 시각화"""
    if words is None:
        words = list(model.wv.key_to_index.keys())[:50]

    # 벡터 추출
    vectors = np.array([model.wv[word] for word in words])

    # t-SNE 차원 축소
    tsne = TSNE(
        n_components=2,
        random_state=42,
        perplexity=min(30, len(words)-1),
        n_iter=1000
    )
    vectors_2d = tsne.fit_transform(vectors)

    # 시각화
    fig, ax = plt.subplots(figsize=(12, 8))
    ax.scatter(vectors_2d[:, 0], vectors_2d[:, 1], alpha=0.7)

    for i, word in enumerate(words):
        ax.annotate(word, (vectors_2d[i, 0], vectors_2d[i, 1]),
                   fontsize=9, ha='center', va='bottom')

    ax.set_title("Word2Vec Embeddings t-SNE Visualization")
    plt.tight_layout()
    plt.savefig('word2vec_tsne.png', dpi=150, bbox_inches='tight')
    plt.show()

visualize_embeddings(model)

# 사전 훈련된 GloVe 벡터 사용
# pip install torchtext

from torchtext.vocab import GloVe

# GloVe 다운로드 및 로드
glove = GloVe(name='6B', dim=100)

# 단어 벡터 접근
vector = glove['computer']
print("'computer' GloVe 벡터 (처음 5차원):", vector[:5].numpy())

# 코사인 유사도 계산
def cosine_similarity(v1, v2):
    return torch.nn.functional.cosine_similarity(
        v1.unsqueeze(0), v2.unsqueeze(0)
    ).item()

import torch
words_to_compare = [('king', 'queen'), ('man', 'woman'), ('Paris', 'France')]
for w1, w2 in words_to_compare:
    if w1 in glove.stoi and w2 in glove.stoi:
        sim = cosine_similarity(glove[w1], glove[w2])
        print(f"sim('{w1}', '{w2}') = {sim:.4f}")

import torch
import torch.nn as nn

class SimpleRNN(nn.Module):
    """기본 RNN 구현"""

    def __init__(self, input_size, hidden_size, output_size):
        super().__init__()
        self.hidden_size = hidden_size

        # 입력 -> 은닉 상태 변환
        self.input_to_hidden = nn.Linear(input_size + hidden_size, hidden_size)
        # 은닉 상태 -> 출력 변환
        self.hidden_to_output = nn.Linear(hidden_size, output_size)

    def forward(self, x, hidden=None):
        """
        x: (seq_len, batch, input_size)
        hidden: (batch, hidden_size) 초기 은닉 상태
        """
        batch_size = x.size(1)

        if hidden is None:
            hidden = torch.zeros(batch_size, self.hidden_size)

        outputs = []
        for t in range(x.size(0)):
            # 입력과 이전 은닉 상태를 결합
            combined = torch.cat([x[t], hidden], dim=1)
            # 새로운 은닉 상태 계산
            hidden = torch.tanh(self.input_to_hidden(combined))
            # 출력 계산
            output = self.hidden_to_output(hidden)
            outputs.append(output)

        return torch.stack(outputs, dim=0), hidden

# PyTorch 내장 RNN 사용
rnn = nn.RNN(
    input_size=50,
    hidden_size=128,
    num_layers=2,
    batch_first=True,
    dropout=0.3,
    bidirectional=True  # 양방향 RNN
)

# 임의 입력으로 테스트
x = torch.randn(32, 20, 50)  # (batch, seq_len, features)
output, hidden = rnn(x)
print("출력 형태:", output.shape)   # (32, 20, 256) - 양방향이므로 128*2
print("은닉 상태 형태:", hidden.shape)  # (4, 32, 128) - 2레이어 * 2방향

class LSTMCell(nn.Module):
    """LSTM 셀 수동 구현 - 게이트 메커니즘 이해를 위해"""

    def __init__(self, input_size, hidden_size):
        super().__init__()
        self.hidden_size = hidden_size

        # 4개의 게이트를 한 번에 계산 (효율성)
        # forget gate, input gate, cell gate, output gate
        self.gates = nn.Linear(input_size + hidden_size, 4 * hidden_size)

    def forward(self, x, state=None):
        """
        x: (batch, input_size)
        state: (h, c) 이전 은닉 상태와 셀 상태
        """
        batch_size = x.size(0)

        if state is None:
            h = torch.zeros(batch_size, self.hidden_size)
            c = torch.zeros(batch_size, self.hidden_size)
        else:
            h, c = state

        # 입력과 이전 은닉 상태 결합
        combined = torch.cat([x, h], dim=1)

        # 4개 게이트 계산
        gates = self.gates(combined)

        # 게이트 분리
        f_gate, i_gate, g_gate, o_gate = gates.chunk(4, dim=1)

        # 각 게이트에 활성화 함수 적용
        f = torch.sigmoid(f_gate)   # forget gate: 이전 정보 얼마나 잊을지
        i = torch.sigmoid(i_gate)   # input gate: 새 정보 얼마나 저장할지
        g = torch.tanh(g_gate)      # cell gate: 새로운 후보 정보
        o = torch.sigmoid(o_gate)   # output gate: 얼마나 출력할지

        # 새로운 셀 상태: 일부 잊고 + 새 정보 추가
        new_c = f * c + i * g

        # 새로운 은닉 상태
        new_h = o * torch.tanh(new_c)

        return new_h, (new_h, new_c)

# PyTorch 내장 LSTM을 이용한 감성 분석 모델
class SentimentLSTM(nn.Module):
    """LSTM 기반 감성 분석 모델"""

    def __init__(self, vocab_size, embed_dim, hidden_dim, num_classes, num_layers=2, dropout=0.3):
        super().__init__()

        self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0)
        self.lstm = nn.LSTM(
            embed_dim,
            hidden_dim,
            num_layers=num_layers,
            batch_first=True,
            dropout=dropout,
            bidirectional=True
        )
        self.dropout = nn.Dropout(dropout)
        # 양방향이므로 hidden_dim * 2
        self.classifier = nn.Sequential(
            nn.Linear(hidden_dim * 2, hidden_dim),
            nn.ReLU(),
            nn.Dropout(dropout),
            nn.Linear(hidden_dim, num_classes)
        )

    def forward(self, x, lengths=None):
        """
        x: (batch, seq_len) 토큰 인덱스
        lengths: 각 시퀀스의 실제 길이 (패딩 고려)
        """
        # 임베딩
        embedded = self.dropout(self.embedding(x))  # (batch, seq, embed)

        # 패딩된 시퀀스 처리
        if lengths is not None:
            packed = nn.utils.rnn.pack_padded_sequence(
                embedded, lengths, batch_first=True, enforce_sorted=False
            )
            lstm_out, (hidden, cell) = self.lstm(packed)
            lstm_out, _ = nn.utils.rnn.pad_packed_sequence(lstm_out, batch_first=True)
        else:
            lstm_out, (hidden, cell) = self.lstm(embedded)

        # 마지막 은닉 상태 (양방향)
        # hidden: (num_layers * 2, batch, hidden_dim)
        forward_hidden = hidden[-2]   # 마지막 레이어의 순방향
        backward_hidden = hidden[-1]  # 마지막 레이어의 역방향
        combined = torch.cat([forward_hidden, backward_hidden], dim=1)

        # 분류
        output = self.classifier(self.dropout(combined))
        return output

class GRUCell(nn.Module):
    """GRU 셀 수동 구현"""

    def __init__(self, input_size, hidden_size):
        super().__init__()
        self.hidden_size = hidden_size

        # Reset gate
        self.reset_gate = nn.Linear(input_size + hidden_size, hidden_size)
        # Update gate
        self.update_gate = nn.Linear(input_size + hidden_size, hidden_size)
        # New gate
        self.new_gate_input = nn.Linear(input_size, hidden_size)
        self.new_gate_hidden = nn.Linear(hidden_size, hidden_size)

    def forward(self, x, h=None):
        batch_size = x.size(0)

        if h is None:
            h = torch.zeros(batch_size, self.hidden_size)

        combined = torch.cat([x, h], dim=1)

        # 리셋 게이트: 이전 은닉 상태 얼마나 리셋할지
        r = torch.sigmoid(self.reset_gate(combined))

        # 업데이트 게이트: 얼마나 업데이트할지
        z = torch.sigmoid(self.update_gate(combined))

        # 새로운 후보 상태
        n = torch.tanh(
            self.new_gate_input(x) + r * self.new_gate_hidden(h)
        )

        # 새로운 은닉 상태
        new_h = (1 - z) * n + z * h

        return new_h

class Encoder(nn.Module):
    """Seq2Seq 인코더"""

    def __init__(self, vocab_size, embed_dim, hidden_dim, num_layers=2, dropout=0.5):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0)
        self.lstm = nn.LSTM(embed_dim, hidden_dim, num_layers,
                           batch_first=True, dropout=dropout)
        self.dropout = nn.Dropout(dropout)

    def forward(self, src):
        embedded = self.dropout(self.embedding(src))
        outputs, (hidden, cell) = self.lstm(embedded)
        return outputs, hidden, cell

class Decoder(nn.Module):
    """Seq2Seq 디코더"""

    def __init__(self, vocab_size, embed_dim, hidden_dim, num_layers=2, dropout=0.5):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0)
        self.lstm = nn.LSTM(embed_dim, hidden_dim, num_layers,
                           batch_first=True, dropout=dropout)
        self.fc = nn.Linear(hidden_dim, vocab_size)
        self.dropout = nn.Dropout(dropout)

    def forward(self, trg, hidden, cell):
        # trg: (batch,) 현재 토큰
        trg = trg.unsqueeze(1)  # (batch, 1)
        embedded = self.dropout(self.embedding(trg))  # (batch, 1, embed)
        output, (hidden, cell) = self.lstm(embedded, (hidden, cell))
        prediction = self.fc(output.squeeze(1))  # (batch, vocab_size)
        return prediction, hidden, cell

class Seq2Seq(nn.Module):
    """Seq2Seq 번역 모델"""

    def __init__(self, encoder, decoder, device):
        super().__init__()
        self.encoder = encoder
        self.decoder = decoder
        self.device = device

    def forward(self, src, trg, teacher_forcing_ratio=0.5):
        """
        src: (batch, src_len) 소스 시퀀스
        trg: (batch, trg_len) 타겟 시퀀스
        teacher_forcing_ratio: 티처 포싱 확률
        """
        batch_size = src.shape[0]
        trg_len = trg.shape[1]
        trg_vocab_size = self.decoder.fc.out_features

        outputs = torch.zeros(batch_size, trg_len, trg_vocab_size).to(self.device)

        # 인코더 실행
        encoder_outputs, hidden, cell = self.encoder(src)

        # 디코더의 첫 입력: <SOS> 토큰
        input = trg[:, 0]

        for t in range(1, trg_len):
            output, hidden, cell = self.decoder(input, hidden, cell)
            outputs[:, t] = output

            # 티처 포싱: 훈련 시 실제 레이블 또는 예측값 사용
            teacher_force = torch.rand(1).item() < teacher_forcing_ratio
            top1 = output.argmax(1)
            input = trg[:, t] if teacher_force else top1

        return outputs

class BahdanauAttention(nn.Module):
    """Bahdanau (Additive) Attention"""

    def __init__(self, hidden_dim):
        super().__init__()
        self.W_enc = nn.Linear(hidden_dim * 2, hidden_dim)  # 양방향 인코더
        self.W_dec = nn.Linear(hidden_dim, hidden_dim)
        self.v = nn.Linear(hidden_dim, 1)

    def forward(self, decoder_hidden, encoder_outputs):
        """
        decoder_hidden: (batch, hidden) 디코더의 현재 은닉 상태
        encoder_outputs: (batch, src_len, hidden*2) 모든 인코더 출력
        """
        src_len = encoder_outputs.shape[1]

        # 디코더 은닉 상태를 src_len만큼 반복
        decoder_hidden = decoder_hidden.unsqueeze(1).repeat(1, src_len, 1)

        # 에너지 계산
        energy = torch.tanh(
            self.W_enc(encoder_outputs) + self.W_dec(decoder_hidden)
        )

        # 어텐션 점수
        attention = self.v(energy).squeeze(2)  # (batch, src_len)
        attention_weights = torch.softmax(attention, dim=1)

        # 컨텍스트 벡터 = 가중합
        context = torch.bmm(
            attention_weights.unsqueeze(1),  # (batch, 1, src_len)
            encoder_outputs                  # (batch, src_len, hidden*2)
        ).squeeze(1)  # (batch, hidden*2)

        return context, attention_weights

class LuongAttention(nn.Module):
    """Luong (Multiplicative) Attention"""

    def __init__(self, hidden_dim, method='dot'):
        super().__init__()
        self.method = method

        if method == 'general':
            self.W = nn.Linear(hidden_dim, hidden_dim)
        elif method == 'concat':
            self.W = nn.Linear(hidden_dim * 2, hidden_dim)
            self.v = nn.Linear(hidden_dim, 1)

    def forward(self, decoder_hidden, encoder_outputs):
        """
        decoder_hidden: (batch, hidden)
        encoder_outputs: (batch, src_len, hidden)
        """
        if self.method == 'dot':
            # 내적
            score = torch.bmm(
                encoder_outputs,
                decoder_hidden.unsqueeze(2)
            ).squeeze(2)

        elif self.method == 'general':
            # W * h_encoder
            energy = self.W(encoder_outputs)
            score = torch.bmm(
                energy,
                decoder_hidden.unsqueeze(2)
            ).squeeze(2)

        elif self.method == 'concat':
            decoder_expanded = decoder_hidden.unsqueeze(1).expand_as(encoder_outputs)
            energy = torch.tanh(self.W(torch.cat([decoder_expanded, encoder_outputs], dim=2)))
            score = self.v(energy).squeeze(2)

        attention_weights = torch.softmax(score, dim=1)
        context = torch.bmm(attention_weights.unsqueeze(1), encoder_outputs).squeeze(1)

        return context, attention_weights

class SelfAttention(nn.Module):
    """Self-Attention 구현"""

    def __init__(self, embed_dim, num_heads=8, dropout=0.1):
        super().__init__()
        assert embed_dim % num_heads == 0

        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.head_dim = embed_dim // num_heads
        self.scale = self.head_dim ** -0.5

        # Query, Key, Value 프로젝션
        self.W_q = nn.Linear(embed_dim, embed_dim)
        self.W_k = nn.Linear(embed_dim, embed_dim)
        self.W_v = nn.Linear(embed_dim, embed_dim)
        self.W_o = nn.Linear(embed_dim, embed_dim)

        self.dropout = nn.Dropout(dropout)

    def forward(self, x, mask=None):
        """
        x: (batch, seq_len, embed_dim)
        mask: (batch, seq_len, seq_len) 어텐션 마스크
        """
        batch_size, seq_len, _ = x.shape

        # Query, Key, Value 계산
        Q = self.W_q(x)  # (batch, seq, embed)
        K = self.W_k(x)
        V = self.W_v(x)

        # 멀티헤드를 위해 형태 변환
        Q = Q.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
        K = K.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
        V = V.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
        # Q, K, V: (batch, num_heads, seq_len, head_dim)

        # Scaled Dot-Product Attention
        scores = torch.matmul(Q, K.transpose(-2, -1)) * self.scale
        # scores: (batch, num_heads, seq_len, seq_len)

        if mask is not None:
            scores = scores.masked_fill(mask == 0, float('-inf'))

        attention_weights = torch.softmax(scores, dim=-1)
        attention_weights = self.dropout(attention_weights)

        # 어텐션 적용
        attended = torch.matmul(attention_weights, V)
        # attended: (batch, num_heads, seq_len, head_dim)

        # 헤드 결합
        attended = attended.transpose(1, 2).contiguous()
        attended = attended.view(batch_size, seq_len, self.embed_dim)

        # 출력 프로젝션
        output = self.W_o(attended)

        return output, attention_weights

import torch
import torch.nn as nn
import math

class PositionalEncoding(nn.Module):
    """사인/코사인 Positional Encoding"""

    def __init__(self, embed_dim, max_len=5000, dropout=0.1):
        super().__init__()
        self.dropout = nn.Dropout(p=dropout)

        # PE 행렬 계산
        pe = torch.zeros(max_len, embed_dim)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(
            torch.arange(0, embed_dim, 2).float() * (-math.log(10000.0) / embed_dim)
        )

        pe[:, 0::2] = torch.sin(position * div_term)  # 짝수 인덱스: sin
        pe[:, 1::2] = torch.cos(position * div_term)  # 홀수 인덱스: cos

        pe = pe.unsqueeze(0)  # (1, max_len, embed_dim)
        self.register_buffer('pe', pe)

    def forward(self, x):
        """x: (batch, seq_len, embed_dim)"""
        x = x + self.pe[:, :x.size(1), :]
        return self.dropout(x)

class LearnablePositionalEncoding(nn.Module):
    """학습 가능한 Positional Encoding (BERT 스타일)"""

    def __init__(self, embed_dim, max_len=512):
        super().__init__()
        self.pe = nn.Embedding(max_len, embed_dim)

    def forward(self, x):
        batch_size, seq_len, _ = x.shape
        positions = torch.arange(seq_len, device=x.device).unsqueeze(0)
        return x + self.pe(positions)

class MultiHeadAttention(nn.Module):
    """Multi-Head Attention"""

    def __init__(self, embed_dim, num_heads, dropout=0.1):
        super().__init__()
        assert embed_dim % num_heads == 0

        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.head_dim = embed_dim // num_heads
        self.scale = self.head_dim ** -0.5

        self.W_q = nn.Linear(embed_dim, embed_dim, bias=False)
        self.W_k = nn.Linear(embed_dim, embed_dim, bias=False)
        self.W_v = nn.Linear(embed_dim, embed_dim, bias=False)
        self.W_o = nn.Linear(embed_dim, embed_dim)
        self.dropout = nn.Dropout(dropout)

    def split_heads(self, x):
        """(batch, seq, embed) -> (batch, heads, seq, head_dim)"""
        batch, seq, _ = x.shape
        x = x.view(batch, seq, self.num_heads, self.head_dim)
        return x.transpose(1, 2)

    def forward(self, query, key, value, mask=None):
        batch_size = query.shape[0]

        Q = self.split_heads(self.W_q(query))
        K = self.split_heads(self.W_k(key))
        V = self.split_heads(self.W_v(value))

        # Scaled Dot-Product Attention
        scores = torch.matmul(Q, K.transpose(-2, -1)) * self.scale

        if mask is not None:
            scores = scores.masked_fill(mask == 0, float('-inf'))

        attn_weights = self.dropout(torch.softmax(scores, dim=-1))

        output = torch.matmul(attn_weights, V)
        output = output.transpose(1, 2).contiguous().view(batch_size, -1, self.embed_dim)

        return self.W_o(output), attn_weights

class FeedForward(nn.Module):
    """Position-wise Feed-Forward Network"""

    def __init__(self, embed_dim, ff_dim, dropout=0.1):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(embed_dim, ff_dim),
            nn.GELU(),  # 논문에서는 ReLU, BERT/GPT에서는 GELU 주로 사용
            nn.Dropout(dropout),
            nn.Linear(ff_dim, embed_dim),
            nn.Dropout(dropout)
        )

    def forward(self, x):
        return self.net(x)

class EncoderLayer(nn.Module):
    """Transformer Encoder Layer"""

    def __init__(self, embed_dim, num_heads, ff_dim, dropout=0.1):
        super().__init__()
        self.self_attention = MultiHeadAttention(embed_dim, num_heads, dropout)
        self.feed_forward = FeedForward(embed_dim, ff_dim, dropout)
        self.norm1 = nn.LayerNorm(embed_dim)
        self.norm2 = nn.LayerNorm(embed_dim)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x, src_mask=None):
        # Self-Attention + Add & Norm (Pre-LN 스타일)
        attn_output, _ = self.self_attention(x, x, x, src_mask)
        x = self.norm1(x + self.dropout(attn_output))

        # Feed-Forward + Add & Norm
        ff_output = self.feed_forward(x)
        x = self.norm2(x + ff_output)

        return x

class DecoderLayer(nn.Module):
    """Transformer Decoder Layer"""

    def __init__(self, embed_dim, num_heads, ff_dim, dropout=0.1):
        super().__init__()
        self.self_attention = MultiHeadAttention(embed_dim, num_heads, dropout)
        self.cross_attention = MultiHeadAttention(embed_dim, num_heads, dropout)
        self.feed_forward = FeedForward(embed_dim, ff_dim, dropout)
        self.norm1 = nn.LayerNorm(embed_dim)
        self.norm2 = nn.LayerNorm(embed_dim)
        self.norm3 = nn.LayerNorm(embed_dim)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x, memory, src_mask=None, tgt_mask=None):
        # Masked Self-Attention (자기 자신에 대한 어텐션, 미래 토큰 마스킹)
        self_attn_out, _ = self.self_attention(x, x, x, tgt_mask)
        x = self.norm1(x + self.dropout(self_attn_out))

        # Cross-Attention (인코더 출력에 대한 어텐션)
        cross_attn_out, cross_attn_weights = self.cross_attention(x, memory, memory, src_mask)
        x = self.norm2(x + self.dropout(cross_attn_out))

        # Feed-Forward
        ff_out = self.feed_forward(x)
        x = self.norm3(x + ff_out)

        return x, cross_attn_weights

class Transformer(nn.Module):
    """완전한 Transformer 구현"""

    def __init__(
        self,
        src_vocab_size,
        tgt_vocab_size,
        embed_dim=512,
        num_heads=8,
        num_encoder_layers=6,
        num_decoder_layers=6,
        ff_dim=2048,
        max_len=5000,
        dropout=0.1
    ):
        super().__init__()

        # 임베딩 레이어
        self.src_embedding = nn.Embedding(src_vocab_size, embed_dim)
        self.tgt_embedding = nn.Embedding(tgt_vocab_size, embed_dim)
        self.pos_encoding = PositionalEncoding(embed_dim, max_len, dropout)
        self.embed_scale = math.sqrt(embed_dim)

        # 인코더 스택
        self.encoder_layers = nn.ModuleList([
            EncoderLayer(embed_dim, num_heads, ff_dim, dropout)
            for _ in range(num_encoder_layers)
        ])

        # 디코더 스택
        self.decoder_layers = nn.ModuleList([
            DecoderLayer(embed_dim, num_heads, ff_dim, dropout)
            for _ in range(num_decoder_layers)
        ])

        # 출력 레이어
        self.output_norm = nn.LayerNorm(embed_dim)
        self.output_projection = nn.Linear(embed_dim, tgt_vocab_size)

        # 가중치 초기화
        self._init_weights()

    def _init_weights(self):
        for p in self.parameters():
            if p.dim() > 1:
                nn.init.xavier_uniform_(p)

    def make_causal_mask(self, seq_len, device):
        """자기회귀 마스크 (미래 토큰 마스킹)"""
        mask = torch.tril(torch.ones(seq_len, seq_len, device=device)).bool()
        return mask.unsqueeze(0).unsqueeze(0)  # (1, 1, seq, seq)

    def make_pad_mask(self, x, pad_idx=0):
        """패딩 마스크"""
        return (x != pad_idx).unsqueeze(1).unsqueeze(2)  # (batch, 1, 1, seq)

    def encode(self, src, src_mask=None):
        x = self.pos_encoding(self.src_embedding(src) * self.embed_scale)
        for layer in self.encoder_layers:
            x = layer(x, src_mask)
        return x

    def decode(self, tgt, memory, src_mask=None, tgt_mask=None):
        x = self.pos_encoding(self.tgt_embedding(tgt) * self.embed_scale)
        for layer in self.decoder_layers:
            x, _ = layer(x, memory, src_mask, tgt_mask)
        return self.output_norm(x)

    def forward(self, src, tgt, src_pad_idx=0, tgt_pad_idx=0):
        # 마스크 생성
        src_mask = self.make_pad_mask(src, src_pad_idx)
        tgt_len = tgt.shape[1]
        tgt_pad_mask = self.make_pad_mask(tgt, tgt_pad_idx)
        tgt_causal_mask = self.make_causal_mask(tgt_len, tgt.device)
        tgt_mask = tgt_pad_mask & tgt_causal_mask

        # 인코더
        memory = self.encode(src, src_mask)

        # 디코더
        output = self.decode(tgt, memory, src_mask, tgt_mask)

        # 출력
        logits = self.output_projection(output)

        return logits

# 모델 인스턴스화 및 테스트
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

model = Transformer(
    src_vocab_size=10000,
    tgt_vocab_size=10000,
    embed_dim=512,
    num_heads=8,
    num_encoder_layers=6,
    num_decoder_layers=6,
    ff_dim=2048,
    dropout=0.1
).to(device)

# 파라미터 수 계산
total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
print(f"총 파라미터 수: {total_params:,}")

# 테스트
src = torch.randint(1, 10000, (4, 20)).to(device)
tgt = torch.randint(1, 10000, (4, 18)).to(device)
output = model(src, tgt)
print(f"출력 형태: {output.shape}")  # (4, 18, 10000)

from transformers import (
    BertTokenizer,
    BertModel,
    BertForSequenceClassification,
    BertForTokenClassification,
    BertForQuestionAnswering,
    AdamW,
    get_linear_schedule_with_warmup
)
import torch
from torch.utils.data import Dataset, DataLoader

# 토크나이저 로드
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

# 기본 인코딩
text = "Natural language processing is transforming AI."
encoding = tokenizer(
    text,
    add_special_tokens=True,    # [CLS], [SEP] 추가
    max_length=128,
    padding='max_length',
    truncation=True,
    return_tensors='pt'
)

print("입력 ID:", encoding['input_ids'][0][:10])
print("어텐션 마스크:", encoding['attention_mask'][0][:10])
print("디코딩:", tokenizer.decode(encoding['input_ids'][0]))

# WordPiece 토큰화 확인
complex_words = ["unbelievable", "preprocessing", "transformers", "tokenization"]
for word in complex_words:
    tokens = tokenizer.tokenize(word)
    print(f"{word:20} -> {tokens}")

class SentimentDataset(Dataset):
    """감성 분석 데이터셋"""

    def __init__(self, texts, labels, tokenizer, max_length=128):
        self.texts = texts
        self.labels = labels
        self.tokenizer = tokenizer
        self.max_length = max_length

    def __len__(self):
        return len(self.texts)

    def __getitem__(self, idx):
        encoding = self.tokenizer(
            self.texts[idx],
            max_length=self.max_length,
            padding='max_length',
            truncation=True,
            return_tensors='pt'
        )
        return {
            'input_ids': encoding['input_ids'].squeeze(),
            'attention_mask': encoding['attention_mask'].squeeze(),
            'label': torch.tensor(self.labels[idx], dtype=torch.long)
        }

def train_bert_classifier(
    train_texts,
    train_labels,
    val_texts,
    val_labels,
    num_labels=2,
    epochs=3,
    batch_size=16,
    lr=2e-5
):
    """BERT 감성 분석 파인튜닝"""

    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

    # 데이터셋 생성
    train_dataset = SentimentDataset(train_texts, train_labels, tokenizer)
    val_dataset = SentimentDataset(val_texts, val_labels, tokenizer)

    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
    val_loader = DataLoader(val_dataset, batch_size=batch_size)

    # 모델 로드
    model = BertForSequenceClassification.from_pretrained(
        'bert-base-uncased',
        num_labels=num_labels
    ).to(device)

    # 옵티마이저
    optimizer = AdamW(model.parameters(), lr=lr, weight_decay=0.01)

    # 스케줄러
    total_steps = len(train_loader) * epochs
    scheduler = get_linear_schedule_with_warmup(
        optimizer,
        num_warmup_steps=total_steps // 10,
        num_training_steps=total_steps
    )

    # 훈련 루프
    best_val_acc = 0

    for epoch in range(epochs):
        model.train()
        total_loss = 0
        correct = 0

        for batch in train_loader:
            input_ids = batch['input_ids'].to(device)
            attention_mask = batch['attention_mask'].to(device)
            labels = batch['label'].to(device)

            optimizer.zero_grad()
            outputs = model(
                input_ids=input_ids,
                attention_mask=attention_mask,
                labels=labels
            )

            loss = outputs.loss
            logits = outputs.logits

            loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            optimizer.step()
            scheduler.step()

            total_loss += loss.item()
            preds = logits.argmax(dim=1)
            correct += (preds == labels).sum().item()

        train_acc = correct / len(train_dataset)
        avg_loss = total_loss / len(train_loader)

        # 검증
        model.eval()
        val_correct = 0
        with torch.no_grad():
            for batch in val_loader:
                input_ids = batch['input_ids'].to(device)
                attention_mask = batch['attention_mask'].to(device)
                labels = batch['label'].to(device)

                outputs = model(input_ids=input_ids, attention_mask=attention_mask)
                preds = outputs.logits.argmax(dim=1)
                val_correct += (preds == labels).sum().item()

        val_acc = val_correct / len(val_dataset)
        print(f"Epoch {epoch+1}: Loss={avg_loss:.4f}, Train Acc={train_acc:.4f}, Val Acc={val_acc:.4f}")

        if val_acc > best_val_acc:
            best_val_acc = val_acc
            torch.save(model.state_dict(), 'best_bert_classifier.pt')

    return model

# 감성 분석 예제
texts = [
    "This movie was absolutely fantastic! I loved every moment.",
    "Terrible film. Complete waste of time and money.",
    "The acting was decent but the plot was confusing.",
    "One of the best movies I've seen this year!",
    "I fell asleep halfway through. So boring."
]
labels = [1, 0, 0, 1, 0]  # 1: positive, 0: negative

from transformers import BertForTokenClassification

# NER 레이블
ner_labels = ['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC']
label2id = {label: i for i, label in enumerate(ner_labels)}
id2label = {i: label for i, label in enumerate(ner_labels)}

model = BertForTokenClassification.from_pretrained(
    'bert-base-uncased',
    num_labels=len(ner_labels),
    id2label=id2label,
    label2id=label2id
)

def predict_ner(text, model, tokenizer):
    """NER 예측"""
    model.eval()

    encoding = tokenizer(
        text.split(),
        is_split_into_words=True,
        return_offsets_mapping=True,
        padding=True,
        truncation=True,
        return_tensors='pt'
    )

    with torch.no_grad():
        outputs = model(
            input_ids=encoding['input_ids'],
            attention_mask=encoding['attention_mask']
        )

    predictions = outputs.logits.argmax(dim=2).squeeze().tolist()
    word_ids = encoding.word_ids()

    # 서브워드 토큰 처리
    result = []
    prev_word_id = None
    for pred, word_id in zip(predictions, word_ids):
        if word_id is None or word_id == prev_word_id:
            continue
        word = text.split()[word_id]
        label = id2label[pred]
        result.append((word, label))
        prev_word_id = word_id

    return result

# 사용 예시
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
text = "Barack Obama was born in Hawaii and served as the 44th President of the United States."
# NER 결과 출력 (훈련된 모델 사용 시)
# print(predict_ner(text, model, tokenizer))

from transformers import GPT2LMHeadModel, GPT2Tokenizer
import torch

# GPT-2 로드
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
model = GPT2LMHeadModel.from_pretrained('gpt2')
model.eval()

tokenizer.pad_token = tokenizer.eos_token

def generate_text(
    prompt,
    model,
    tokenizer,
    max_length=200,
    temperature=0.9,
    top_p=0.95,
    top_k=50,
    num_return_sequences=1,
    do_sample=True
):
    """GPT-2 텍스트 생성"""

    inputs = tokenizer(prompt, return_tensors='pt')
    input_ids = inputs['input_ids']

    with torch.no_grad():
        output = model.generate(
            input_ids,
            max_length=max_length,
            temperature=temperature,      # 높을수록 다양한 출력
            top_p=top_p,                  # nucleus sampling
            top_k=top_k,                  # top-k sampling
            num_return_sequences=num_return_sequences,
            do_sample=do_sample,
            pad_token_id=tokenizer.eos_token_id,
            repetition_penalty=1.2        # 반복 방지
        )

    generated_texts = []
    for out in output:
        text = tokenizer.decode(out, skip_special_tokens=True)
        generated_texts.append(text)

    return generated_texts

# 텍스트 생성 예시
prompt = "Artificial intelligence is transforming the world by"
generated = generate_text(prompt, model, tokenizer, max_length=150, temperature=0.8)

for i, text in enumerate(generated):
    print(f"\n생성된 텍스트 {i+1}:")
    print(text)
    print("-" * 50)

# Greedy vs Sampling 비교
def compare_generation_strategies(prompt, model, tokenizer):
    """다양한 생성 전략 비교"""

    input_ids = tokenizer.encode(prompt, return_tensors='pt')

    strategies = {
        "Greedy Search": dict(do_sample=False),
        "Beam Search": dict(do_sample=False, num_beams=5, early_stopping=True),
        "Temperature Sampling": dict(do_sample=True, temperature=0.7),
        "Top-k Sampling": dict(do_sample=True, top_k=50),
        "Top-p (Nucleus) Sampling": dict(do_sample=True, top_p=0.92),
    }

    print(f"프롬프트: {prompt}\n")

    with torch.no_grad():
        for name, params in strategies.items():
            output = model.generate(
                input_ids,
                max_new_tokens=50,
                pad_token_id=tokenizer.eos_token_id,
                **params
            )
            text = tokenizer.decode(output[0], skip_special_tokens=True)
            print(f"[{name}]")
            print(text[len(prompt):].strip())
            print()

compare_generation_strategies(
    "The future of artificial intelligence",
    model,
    tokenizer
)

def zero_shot_classification(text, categories, model_name="gpt2"):
    """Zero-shot 분류"""

    tokenizer = GPT2Tokenizer.from_pretrained(model_name)
    model = GPT2LMHeadModel.from_pretrained(model_name)
    model.eval()

    # 각 카테고리에 대한 로그 확률 계산
    scores = {}

    for category in categories:
        prompt = f"Text: {text}\nCategory: {category}"
        inputs = tokenizer(prompt, return_tensors='pt')

        with torch.no_grad():
            outputs = model(**inputs, labels=inputs['input_ids'])
            # 낮은 perplexity = 더 가능성 높은 카테고리
            scores[category] = -outputs.loss.item()

    best_category = max(scores, key=scores.get)
    return best_category, scores

# Few-shot 학습 예시
few_shot_prompt = """
Classify the sentiment of the following texts:

Text: "This movie was amazing!"
Sentiment: Positive

Text: "I hated every minute of it."
Sentiment: Negative

Text: "The film was okay, nothing special."
Sentiment: Neutral

Text: "Absolutely brilliant performance!"
Sentiment:"""

print("Few-shot 프롬프트:")
print(few_shot_prompt)

# Chain of Thought 프롬프팅
cot_prompt = """
Q: A train travels 120 miles in 2 hours. How long will it take to travel 300 miles?

A: Let me think step by step.
1. First, find the speed: 120 miles / 2 hours = 60 miles per hour
2. Then, calculate the time for 300 miles: 300 miles / 60 mph = 5 hours
Therefore, it will take 5 hours.

Q: If a store sells 3 apples for $2.40, how much does 7 apples cost?

A: Let me think step by step.
"""

from peft import LoraConfig, get_peft_model, TaskType
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

def setup_lora_model(model_name="gpt2", r=8, lora_alpha=32, lora_dropout=0.1):
    """LoRA 모델 설정"""

    # 기본 모델 로드
    model = AutoModelForCausalLM.from_pretrained(
        model_name,
        torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32
    )
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    tokenizer.pad_token = tokenizer.eos_token

    # LoRA 설정
    lora_config = LoraConfig(
        task_type=TaskType.CAUSAL_LM,
        r=r,                          # LoRA 랭크
        lora_alpha=lora_alpha,        # 스케일링 파라미터
        target_modules=["c_attn"],    # 적용할 레이어
        lora_dropout=lora_dropout,
        bias="none"
    )

    # LoRA 모델 생성
    model = get_peft_model(model, lora_config)

    # 훈련 가능한 파라미터 비율 출력
    trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
    total = sum(p.numel() for p in model.parameters())
    print(f"훈련 가능한 파라미터: {trainable:,} ({100 * trainable / total:.2f}%)")

    return model, tokenizer

# LoRA 학습 예시
class InstructionDataset(torch.utils.data.Dataset):
    """지시문 형식 데이터셋"""

    def __init__(self, data, tokenizer, max_length=512):
        self.tokenizer = tokenizer
        self.max_length = max_length

        # 지시문 형식으로 변환
        self.texts = []
        for item in data:
            prompt = f"### Instruction:\n{item['instruction']}\n\n### Response:\n{item['response']}"
            self.texts.append(prompt)

    def __len__(self):
        return len(self.texts)

    def __getitem__(self, idx):
        encoding = self.tokenizer(
            self.texts[idx],
            max_length=self.max_length,
            padding='max_length',
            truncation=True,
            return_tensors='pt'
        )
        input_ids = encoding['input_ids'].squeeze()
        return {
            'input_ids': input_ids,
            'attention_mask': encoding['attention_mask'].squeeze(),
            'labels': input_ids.clone()  # 언어 모델링: 입력 = 레이블
        }

from transformers import AutoTokenizer, AutoModel
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

class SimpleRAG:
    """간단한 RAG 시스템 구현"""

    def __init__(self, embedding_model="sentence-transformers/all-MiniLM-L6-v2"):
        self.tokenizer = AutoTokenizer.from_pretrained(embedding_model)
        self.model = AutoModel.from_pretrained(embedding_model)
        self.knowledge_base = []
        self.embeddings = []

    def mean_pooling(self, model_output, attention_mask):
        """평균 풀링으로 문장 임베딩 계산"""
        token_embeddings = model_output[0]
        input_mask_expanded = attention_mask.unsqueeze(-1).expand(
            token_embeddings.size()
        ).float()
        return torch.sum(token_embeddings * input_mask_expanded, 1) / \
               torch.clamp(input_mask_expanded.sum(1), min=1e-9)

    def encode(self, texts):
        """텍스트를 임베딩 벡터로 변환"""
        encoded = self.tokenizer(
            texts,
            padding=True,
            truncation=True,
            max_length=512,
            return_tensors='pt'
        )

        with torch.no_grad():
            model_output = self.model(**encoded)

        embeddings = self.mean_pooling(model_output, encoded['attention_mask'])
        # L2 정규화
        embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1)
        return embeddings.numpy()

    def add_documents(self, documents):
        """지식베이스에 문서 추가"""
        self.knowledge_base.extend(documents)
        new_embeddings = self.encode(documents)
        if len(self.embeddings) == 0:
            self.embeddings = new_embeddings
        else:
            self.embeddings = np.vstack([self.embeddings, new_embeddings])
        print(f"{len(documents)}개의 문서가 추가되었습니다. 총 {len(self.knowledge_base)}개.")

    def retrieve(self, query, top_k=3):
        """쿼리와 가장 유사한 문서 검색"""
        query_embedding = self.encode([query])
        similarities = cosine_similarity(query_embedding, self.embeddings)[0]
        top_indices = np.argsort(similarities)[::-1][:top_k]

        results = []
        for idx in top_indices:
            results.append({
                'document': self.knowledge_base[idx],
                'similarity': similarities[idx]
            })

        return results

    def answer(self, query, generator_model, generator_tokenizer, top_k=3):
        """RAG: 검색 + 생성"""
        # 관련 문서 검색
        relevant_docs = self.retrieve(query, top_k=top_k)

        # 컨텍스트 구성
        context = "\n".join([doc['document'] for doc in relevant_docs])

        # 프롬프트 구성
        prompt = f"""Based on the following context, answer the question.

Context:
{context}

Question: {query}

Answer:"""

        # 텍스트 생성
        inputs = generator_tokenizer(prompt, return_tensors='pt', max_length=512, truncation=True)
        with torch.no_grad():
            outputs = generator_model.generate(
                inputs['input_ids'],
                max_new_tokens=150,
                temperature=0.7,
                do_sample=True,
                pad_token_id=generator_tokenizer.eos_token_id
            )

        answer = generator_tokenizer.decode(outputs[0], skip_special_tokens=True)
        answer = answer[len(prompt):].strip()

        return answer, relevant_docs

# RAG 시스템 사용 예시
rag = SimpleRAG()

documents = [
    "BERT (Bidirectional Encoder Representations from Transformers) was published by Google in 2018.",
    "GPT-3 has 175 billion parameters and was developed by OpenAI in 2020.",
    "The Transformer architecture was introduced in the paper 'Attention is All You Need' in 2017.",
    "RLHF (Reinforcement Learning from Human Feedback) is used to align language models with human values.",
    "LoRA allows efficient fine-tuning by adding low-rank matrices to pre-trained model weights.",
    "RAG combines information retrieval with text generation for more accurate responses.",
]

rag.add_documents(documents)

query = "What is BERT and when was it created?"
results = rag.retrieve(query, top_k=2)

print(f"\n쿼리: {query}")
print("\n검색된 문서:")
for r in results:
    print(f"  [{r['similarity']:.3f}] {r['document']}")