library(dplyr)
library(tidytext)
library(textstem)
library(stringr)

# Model sentences
text_data <- tibble::tibble(
  text = c(
    "The maize yield was really good this season.",
    "Farmers reported good rainfall and healthy maize crops.",
    "The delayed monsoon affected sowing in several regions.",
    "Despite the challenges, farmers adapted their farming practices."
  )
)

# Settings (default values)
min_len     <- 3L               # Minimum token length
remove_stop <- TRUE             # Remove stopwords
lemmatise   <- TRUE             # Lemmatise words
custom_stop <- character(0)     # Additional user-defined stopwords
top_n       <- 40L              # Number of most frequent words to display

# Default stopword list
qda_stopwords <- unique(c(
  stopwords::stopwords("en"),
  tidytext::stop_words$word,
  c("yes", "no", "na", "n/a", "etc", "also", "like", "really", "thing", "things")
))

# 1. Split the text into lowercase words and keep only alphabetic
#    words with at least the specified minimum length.
tokens <- text_data %>%
  unnest_tokens(word, text) %>%
  filter(str_detect(word, "[a-z]")) %>%
  filter(nchar(word) >= min_len)

# 2. Remove stopwords (default enabled)
if (remove_stop)
  tokens <- tokens %>%
  filter(!word %in% c(qda_stopwords, custom_stop))

# 3. Convert words to their base (lemma) form (default enabled)
if (lemmatise)
  tokens <- tokens %>%
  mutate(word = lemmatize_words(word))

# 4. Count the frequency of each word and compute its relative frequency.
total_tokens <- nrow(tokens)

word_frequency <- tokens %>%
  count(word, sort = TRUE, name = "Count") %>%
  mutate(`Relative frequency (%)` = round(100 * Count / total_tokens, 2))

head(word_frequency, top_n)  

library(syuzhet)
library(sentimentr)
library(stringr)
library(dplyr)

# Unit of analysis: "doc", "sentence" or "segment"
unit <- "sentence"

text <- c("The maize yield was really good this season.",
          "Unfortunately the late rains ruined part of the harvest.")

# --- Build units ---
# "sentence": split on . ! ? ; "doc": whole text; "segment": the coded span text
if (unit == "sentence") {
  units <- unlist(stringr::str_split(text, "(?<=[.!?])\\s+"))
  units <- units[nzchar(trimws(units))]
} else {
  units <- text
}

# Whole-document scores are divided by word count; sentence/segment left as-is
avg_if_doc <- function(score, txt) {
  if (unit != "doc") return(score)
  nw <- stringr::str_count(txt, "\\S+"); nw[is.na(nw) | nw == 0] <- 1L
  score / nw
}

# --- Bing: summed +/-1 lexicon ---
bing  <- avg_if_doc(suppressWarnings(syuzhet::get_sentiment(units, method = "bing")),  units)

# --- AFINN: summed -5..+5 lexicon ---
afinn <- avg_if_doc(suppressWarnings(syuzhet::get_sentiment(units, method = "afinn")), units)

# --- NRC: 8 emotions + polarity; score = positive - negative ---
nrc_mat <- suppressWarnings(syuzhet::get_nrc_sentiment(units))
nrc     <- avg_if_doc(nrc_mat$positive - nrc_mat$negative, units)

# --- VADER: sentence-level valence via sentimentr (negation / valence-shifter
#     aware; replaces the archived 'vader' package). Already length-normalised ---
vader <- sentimentr::sentiment_by(units, by = NULL)$ave_sentiment

# --- Polarity label (lexicons use 0; VADER uses a +/-0.05 neutral dead-band) ---
polarity <- function(score, deadband = 0) dplyr::case_when(
  score >  deadband ~ "Positive",
  score < -deadband ~ "Negative",
  TRUE              ~ "Neutral"
)

data.frame(
  unit           = units,
  bing, afinn, nrc, vader,
  bing_polarity  = polarity(bing),
  vader_polarity = polarity(vader, 0.05)
)