IIS_2023_1/faskhutdinov_idris_lab_7/main.py

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, LSTM, Dense
import numpy as np

# Загрузка и подготовка данных
with open('russian_text.txt', 'r', encoding='cp1251') as file:
    russian_text = file.read()

# Токенизация текста
tokenizer = tf.keras.preprocessing.text.Tokenizer()
tokenizer.fit_on_texts([russian_text])
total_words = len(tokenizer.word_index) + 1

# Преобразование текста в числовые последовательности
input_sequences = []
for line in russian_text.split('n'):
    token_list = tokenizer.texts_to_sequences([line])[0]
    for i in range(1, len(token_list)):
        n_gram_sequence = token_list[:i+1]
        input_sequences.append(n_gram_sequence)

# Создание входных и целевых последовательностей
max_sequence_length = max([len(x) for x in input_sequences])
input_sequences = np.array(tf.keras.preprocessing.sequence.pad_sequences(input_sequences, maxlen=max_sequence_length,
                                                                         padding='pre'))
predictors, label = input_sequences[:, :-1], input_sequences[:, -1]

# Построение и обучение модели
model = Sequential()
model.add(Embedding(total_words, 100, input_length=max_sequence_length-1))
model.add(LSTM(150))
model.add(Dense(total_words, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy', optimizer='adam')
model.fit(predictors, label, epochs=150, batch_size=64)


def generate_text(seed_text, next_words, model, max_sequence_length):
    for _ in range(next_words):
        token_list = tokenizer.texts_to_sequences([seed_text])[0]
        token_list = tf.keras.preprocessing.sequence.pad_sequences([token_list], maxlen=max_sequence_length - 1,
                                                                   padding='pre')
        predicted = np.argmax(model.predict(token_list, verbose=0), axis=-1)

        # Поиск слова, соответствующего предсказанному индексу
        output_word = ""
        for word, index in tokenizer.word_index.items():
            if index == predicted:
                output_word = word
                break

        seed_text += " " + output_word

    return seed_text

# Генерация и сохранение русскоязычного текста
russian_generated_text = generate_text('Абрикосовая', next_words=100, model=model, max_sequence_length=max_sequence_length)
with open('generated_russian_text.txt', 'w', encoding='utf-8') as file:
    file.write(russian_generated_text)



# Загрузка и подготовка данных
with open('english_text.txt', 'r', encoding='utf-8') as file:
    english_text = file.read()

# Токенизация текста
tokenizer = tf.keras.preprocessing.text.Tokenizer()
tokenizer.fit_on_texts([english_text])
total_words = len(tokenizer.word_index) + 1

# Преобразование текста в числовые последовательности
input_sequences = []
for line in english_text.split('n'):
    token_list = tokenizer.texts_to_sequences([line])[0]
    for i in range(1, len(token_list)):
        n_gram_sequence = token_list[:i+1]
        input_sequences.append(n_gram_sequence)

# Создание входных и целевых последовательностей
max_sequence_length = max([len(x) for x in input_sequences])
input_sequences = np.array(tf.keras.preprocessing.sequence.pad_sequences(input_sequences, maxlen=max_sequence_length,
                                                                         padding='pre'))
predictors, label = input_sequences[:, :-1], input_sequences[:, -1]

# Построение и обучение модели
model = Sequential()
model.add(Embedding(total_words, 100, input_length=max_sequence_length-1))
model.add(LSTM(150))
model.add(Dense(total_words, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy', optimizer='adam')
model.fit(predictors, label, epochs=150, batch_size=64)

# Генерация и сохранение англоязычного текста
english_generated_text = generate_text('It', next_words=100, model=model, max_sequence_length=max_sequence_length)
with open('generated_english_text.txt', 'w', encoding='utf-8') as file:
    file.write(english_generated_text)