import spacy from collections import Counter try: nlp = spacy.load("ru_core_news_lg") except OSError: import spacy.cli spacy.cli.download("ru_core_news_lg") nlp = spacy.load("ru_core_news_lg") def analyze_text(text): doc = nlp(text) tokens = [token.text for token in doc] words = [token.text for token in doc if token.is_alpha] unique_words = set(words) stop_words = [token.text for token in doc if token.is_stop] avg_word_length = sum(len(word) for word in words) / len(words) if words else 0 pos_counts = Counter([token.pos_ for token in doc if token.is_alpha]) lemmas = [token.lemma_ for token in doc if token.is_alpha] unique_lemmas = set(lemmas) dependencies = Counter([token.dep_ for token in doc if token.dep_ != ""]) has_noun_chunks = False try: next(doc.noun_chunks, None) has_noun_chunks = True except NotImplementedError: pass entities = [(ent.text, ent.label_) for ent in doc.ents] entity_counts = Counter([ent.label_ for ent in doc.ents]) ttr = len(unique_words) / len(words) if words else 0 def calculate_simplified_mtld(text_tokens, ttr_threshold=0.72): if len(text_tokens) < 10: return 0 segments = [] current_segment = [] for token in text_tokens: current_segment.append(token) current_ttr = len(set(current_segment)) / len(current_segment) if current_ttr <= ttr_threshold and len(current_segment) >= 10: segments.append(current_segment) current_segment = [] if current_segment: segments.append(current_segment) if not segments: return 0 return len(text_tokens) / len(segments) mtld = calculate_simplified_mtld(words) sentences = list(doc.sents) sentence_lengths = [len(sent) for sent in sentences] avg_sentence_length = sum(sentence_lengths) / len(sentences) if sentences else 0 words_per_sentence = len(words) / len(sentences) if sentences else 0 def count_syllables_ru(word): return len([c for c in word.lower() if c in 'аеёиоуыэюя']) syllables = sum(count_syllables_ru(word) for word in words) syllables_per_word = syllables / len(words) if words else 0 flesh_kincaid = 206.835 - 1.3 * words_per_sentence - 60.1 * syllables_per_word long_words = [word for word in words if count_syllables_ru(word) > 4] long_words_percent = len(long_words) / len(words) * 100 if words else 0 sentence_count = len(sentences) question_count = sum(1 for sent in sentences if sent.text.strip().endswith('?')) exclamation_count = sum(1 for sent in sentences if sent.text.strip().endswith('!')) coherence_scores = [] if len(sentences) > 1: for i in range(len(sentences)-1): if len(sentences[i]) > 0 and len(sentences[i+1]) > 0: try: if sentences[i].vector_norm > 0 and sentences[i+1].vector_norm > 0: sim = sentences[i].similarity(sentences[i+1]) coherence_scores.append(sim) except: pass avg_coherence = sum(coherence_scores) / len(coherence_scores) if coherence_scores else 0 analysis_results = { 'basic_stats': { 'total_tokens': len(tokens), 'total_words': len(words), 'unique_words': len(unique_words), 'stop_words': len(stop_words), 'avg_word_length': avg_word_length }, 'morphological_analysis': { 'pos_distribution': {pos: count for pos, count in pos_counts.items()}, 'unique_lemmas': len(unique_lemmas), 'lemma_word_ratio': len(unique_lemmas) / len(unique_words) if unique_words else 0 }, 'syntactic_analysis': { 'dependencies': {dep: count for dep, count in dependencies.most_common(10)}, 'noun_chunks': has_noun_chunks }, 'named_entities': { 'total_entities': len(entities), 'entity_types': {label: count for label, count in entity_counts.items()} }, 'lexical_diversity': { 'ttr': ttr, 'mtld': mtld }, 'text_structure': { 'sentence_count': sentence_count, 'avg_sentence_length': avg_sentence_length, 'question_sentences': question_count, 'exclamation_sentences': exclamation_count }, 'readability': { 'words_per_sentence': words_per_sentence, 'syllables_per_word': syllables_per_word, 'flesh_kincaid_score': flesh_kincaid, 'long_words_percent': long_words_percent }, 'semantic_coherence': { 'avg_coherence_score': avg_coherence } } return analysis_results def show_text_analysis(analysis): print("\n📊 TEXT ANALYSIS") print("\n=== BASIC STATISTICS ===") print(f"- Total tokens: {analysis['basic_stats']['total_tokens']}") print(f"- Total words: {analysis['basic_stats']['total_words']}") print(f"- Unique words: {analysis['basic_stats']['unique_words']}") print(f"- Stop words: {analysis['basic_stats']['stop_words']}") print(f"- Average word length: {analysis['basic_stats']['avg_word_length']:.2f} characters") print("\n=== MORPHOLOGICAL ANALYSIS ===") print("- POS distribution:") for pos, count in analysis['morphological_analysis']['pos_distribution'].items(): print(f" • {pos}: {count}") print(f"- Unique lemmas: {analysis['morphological_analysis']['unique_lemmas']}") print("\n=== SYNTACTIC ANALYSIS ===") print("- Syntactic dependencies (top-5):") for i, (dep, count) in enumerate(analysis['syntactic_analysis']['dependencies'].items()): if i >= 5: break print(f" • {dep}: {count}") print("\n=== NAMED ENTITIES ===") print(f"- Total entities: {analysis['named_entities']['total_entities']}") print("\n=== LEXICAL DIVERSITY ===") print(f"- TTR (type-token ratio): {analysis['lexical_diversity']['ttr']:.3f}") print(f"- MTLD (simplified): {analysis['lexical_diversity']['mtld']:.2f}") print("\n=== TEXT STRUCTURE ===") print(f"- Sentence count: {analysis['text_structure']['sentence_count']}") print(f"- Average sentence length: {analysis['text_structure']['avg_sentence_length']:.2f} tokens") print("\n=== READABILITY ===") print(f"- Flesch-Kincaid score: {analysis['readability']['flesh_kincaid_score']:.2f}") print(f"- Long words percentage: {analysis['readability']['long_words_percent']:.2f}%") print(f"\n=== SEMANTIC COHERENCE ===") print(f"- Average coherence between sentences: {analysis['semantic_coherence']['avg_coherence_score']:.3f}")