Как построить генеративную модель вывода стиля

import os
import sys
import glob
import itertools
import numpy as np
import pandas as pd

from nltk.tokenize import WordPunctTokenizer
from nltk.stem import WordNetLemmatizer

from gensim.models.phrases import Phrases
from gensim.models.word2vec import Word2Vec

import tensorflow as tf
from tensorflow import keras

# you should be familiar with this
%run utils.py

def load_phrase_detector_model(fname, reduce_size=False):
    phrases = Phrases.load(fname)

    print(f"Loading complete")
    return phrases.freeze() if reduce_size else phrases

# load a phrase detector model
phrase_model_path = "malaz_phrase_detector"
phrases = load_phrase_detector_model(phrase_model_path, reduce_size=True)

sentences_iterator = CustomPathLineSentences('Books', include_phrase=True,
                                             phrase_model=phrases)

# load word2vec model
word2vec_path = "malaz_word2vec.bin"
word2vec = Word2Vec.load(word2vec_path)
word2vec = word2vec.wv

print("Setup complete")

def preprocess_texts(sentences_iterator):
    text = []
    for sentence in sentences_iterator:
        # remember each sentence is a list of tokens
        # punctuation included
        text.extend(sentence)

    print(f"Total number of words (Phrases included) {len(text)}")

    return text

text = preprocess_texts(sentences_iterator)

vocab = sorted(list(set(text)))
word_indices = {word: idx for idx, word in enumerate(vocab)}
indices_word = {idx: word for idx, word in enumerate(vocab)}

print(f"Total number of unique words: {len(vocab)}")

total_words = len(text)
total_sentences = len(sentences_iterator)
avg_word_sentences = total_words / total_sentences

print(f"Avg word per sentence: {avg_word_sentences}")

maxlen = int(avg_word_sentences + 40)
step = 3
sentences = []
next_word_target []

for idx in range(0, len(text) - maxlen, step):
    sentences.append(text[idx: idx + maxlen])
    next_word_target.append(text[idx + maxlen])

# no longer need
del text
print(len(sentences), len(next_word_target))

def cached_dataset(sentences, next_word_target, word2vec, word_indices):
    """A factory function, it returns a generator function
    that has various variables cached in it scope that will
    be unaffected by the outer scope.

    Doing this will enable the iterator function to be called
    without the need for arguments and without the function
    to request the variables from the outer scopes, because
    they are stored in it internals."""

    def generator():
        """An iterator function, it simply iterates through our earlier
        created dataset, changing the words in each sentence to its
        word vector representatives"""


        # words not available in the word vector
        # will be given a default word vectors where
        # every dimension equals to zero
        unknown_word = np.zeros(shape=(word_vector.vector_size,),
                                dtype=np.float32)

        for sentence, target_word in itertools.zip_longest(sentences, next_word_target):
            # create an dummy array of shape (len(sentence), embed_dim)
            # this is the word vector representation
            data = np.zeros(shape=(maxlen, word2vec.vector_size),
                            dtype=np.float32)

            # fills in the dummy array with the real word vector
            # values.
            for idx, word in enumerate(itertools.islice(sentence, None)):
                if word in word2vec:
                    data[idx] = word_vector[word]
                else:
                    data[idx] = unknown_word

            # create the target array
            target = np.array([word_indices[target_word]], dtype=np.int32)

            yield (tf.convert_to_tensor(data, dtype=tf.float32),
                   tf.convert_to_tensor(target, dtype=tf.int32))
    return generator

gen = cached_dataset(sentences, next_word_target, word2vec, word_indices)

# create a tensor dataset generator using 
# the Dataset API 
dataset_generator = tf.data.Dataset.from_generator(
    gen, 
    output_signature=(tf.TensorSpec(shape=(maxlen, word2vec.vector_size),
                                    dtype=tf.float32),
                      tf.TensorSpec(shape=(1,), dtype=tf.int32)))

num_neuron = 500
model = keras.models.Sequential()
model.add(keras.layers.Input(shape=(maxlen, word2vec.vector_size)))
model.add(keras.layers.LSTM(num_neuron, return_sequences=True))
model.add(keras.layers.LSTM(num_neuron, return_sequences=True))
model.add(keras.layers.LSTM(num_neuron))

model.add(keras.layers.Dense(len(vocab), activation='softmax'))

optimizer = keras.optimizer.Adam(learning_rate=0.001)
model.compile(loss='sparse_categorical_crossentropy',
              optimizer=optimizer)
model.summary()

# define a checkpoint callback [you will really need this]
model_cb = keras.callbacks.ModelCheckpoint("model.h5", monitor='loss')

# GET READY TO RECIEVE THE GREATEST SHOCK
# OF YOUR LIFE WHEN YOU RUN THIS CODE

# Tell me what you see when you execute the code
# what did it display (You will know what I am
# talking about when you see it)
epochs = 100
batch_sixe = 32
num_workers = 4
model.fit(dataset_generator.repeat(-1).batch(batch_size).prefetch(5),
          steps_per_epoch=len(sentences) // batch_size,
          batch_size=batch_size, epochs=epochs,
          workers=num_workers, callbacks=[model_cb])

def sample(preds, temperature=1.0):
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)

    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)

    return np.argmax(probas)

text = """
it was not of Jaghut construction, that it had arisen beside the three Jaghut towers of its own accord, in answer to a law unfathomable to god and mortal alike. Arisen to await the coming of those whom it would imprison for eternity. Creatures of deadly power.
"""
text = sentences_iterator.clean_token_words(text)
text = text[:maxlen]
generated = []
generated.extend(text)
sys.stdout.write(generate)

unknown_word = np.zeros(shape=(word_vector.vector_size,),
                        dtype=np.float32)

for temp in np.arange(0.1, 0.5, 0.1):
    for i in range(400):
        data = np.zeros(shape=(1, maxlen, word2vec.vector_size),
                        dtype=np.float32)

        for t, word in enumerate(text):
            if word in word2vec:
                data[0, t] = word2vec[word]
            else:
                data[0, t] = unknown_word

        preds = model.predict(data)
        next_index = sample(preds.ravel(), t)  # optimal value 0.2 to 0.4, test
        next_word = indices_word[next_index]
        generated += next_word

        text[1:].append(next_word)
        sys.stdout.write(next_word)
        sys.stdout.flush()