rm(list = setdiff(ls(), lsf.str()))
BasePath = "/Users/rizwan/Library/Mobile Documents/com~apple~CloudDocs/PhD/initialexperiments/"
readPath =  paste0(BasePath ,"4-Merged Dataset/")
writePath = paste0(BasePath ,"5-Merged Dataset After Filteration/")

dataSetName = list.files(path = readPath)
for (names in 1:NROW(dataSetName)) {
  cat(names, "in process\n")
  CompletePathAndFile = paste0(readPath, dataSetName[names])
  mydata = read.csv(CompletePathAndFile, header = TRUE)
  mydata <- unique(mydata)
  inconsistant <- which(duplicated(mydata[, -ncol(mydata)]))
  inconsistant2 <-
    which(duplicated(mydata[, -ncol(mydata)], fromLast = TRUE))
  inconsistant <- append(inconsistant, inconsistant2)
  mydata <- mydata[-inconsistant,]
  if (nrow(mydata) >= 100) {
    if (prop.table(table(mydata$BUG))[[1]] >= 0.1 &&
        prop.table(table(mydata$BUG))[[1]] <= 0.9) {
      write.csv(mydata,
                row.names = FALSE,
                file =  paste0(writePath,
                               dataSetName[names]))
    }
  }
}