package org.cloudbus.cloudsim.codes.reinforcementLearning.sarsa;

import org.apache.commons.io.FileUtils;

import java.io.File;
import java.util.*;

/**
 * ClassName SSARSA
 * Description
 * Create by HW
 * Date 2021/11/23 20:44
 */
//其实此处就为这个agent的东西
class CliffState{
    int x;
    int y;
    int length;
    int width;
    int rew;
    int ret;//reward total总共的奖励
    public CliffState(int w, int l){
        reset();
        width = w;
        length = l;

    }

    public void reset(){
        x = 0;
        y = 0;
        rew = 0;
        ret = 0;
    }
//群体智能的动作，然后给出相关的奖励函数reward
    public void action(String a){
        if(a.equals("up")){
            up();
        }else if(a.equals("down")){
            down();
        }else if(a.equals("left")){
            left();
        }else if(a.equals("right")){
            right();
        }
        reward();
    }

    public void up(){
        if(y < (length-1)){
            y++;
        }
    }

    public void down(){
        if(y > 0){
            y--;
        }
    }

    public void right(){
        if(x < (width-1)){
            x++;
        }
    }

    public void left(){
        if(x > 0){
            x--;
        }
    }

    public void reward(){
        if((y==0) && (x>0) && (x<(width-1))){
            reset();
            rew = -100;
        }else{
            rew = -1;
        }
        ret += rew;
    }

    public int getReward(){
        return rew;
    }

    //结束的标志
    public boolean terminate(){
        return (x > 0 && (y==0));
    }

    //获得这个悬崖的状态
    public String getState(){
        return String.format("(%d, %d)", x, y);
    }
}

public class CliffWalking{
    int w;
    int l;

    public CliffWalking(int width, int length){
        w = width;
        l = length;
    }
//HashMap<String, Double> qa 为行为概率
    //该函数主要是对获得该悬崖状态的一个探讨，进行一个探索
    public String etaGreedy(HashMap<String, Double> qa, double eta){
        Random rand = new Random();
        Double maxValue = Double.NEGATIVE_INFINITY;
        int maxCount = 0;
        int totalCount = 0;
        LinkedList<String> actions = new LinkedList<String>();
        LinkedList<Double> probs = new LinkedList<Double>();//建立相关概率列表
        for(String action : qa.keySet()){
            totalCount++;
            Double currValue = qa.get(action);
            // System.out.printf("%f vs. %f\n", currValue, maxValue);
            int compare = currValue.compareTo(maxValue);
            if(compare > 0){
                maxValue = currValue;
                maxCount = 1;
            }else if(compare == 0){
                // System.out.println("duplicate max");
                maxCount++;
            }
        }
        //System.out.println("最大值maxValue为：" + maxValue + "最大的计数maxCount:" + maxCount);
        //下面进行动作的探索和这个动作的开发
        Double exploreProb = eta/totalCount;
        Double greedyProb = (1.0 - eta)/maxCount + exploreProb;
        Double oldProb = 0.0;
        //System.out.printf("eV: %f\n gV: %f\nmaxCount: %d\n max: %f\n", exploreProb, greedyProb, maxCount, maxValue);
        for(String action : qa.keySet()){
            Double currValue = qa.get(action);
            if(currValue.compareTo(maxValue)==0){
                oldProb += greedyProb;
            }else{
                oldProb += exploreProb;
            }
            //System.out.printf("%f\n", oldProb);
            probs.add(oldProb);
            actions.add(action);
        }
        double r = rand.nextDouble();
        for(int i = 0; i < totalCount; i++){
            if(r < probs.get(i)){
                return actions.get(i);
            }
        }
        return actions.get(totalCount-1);
    }
//此时是获得最大的MaxQ,和下一阶段的这个动作产生的reward进行比较
    public Double getMaxQAV(HashMap<String, Double> qa){
        Double maxValue = Double.NEGATIVE_INFINITY;
        String maxAction = null;
        for(String action : qa.keySet()){
            Double currValue = qa.get(action);
            int compare = currValue.compareTo(maxValue);
            if(compare > 0){
                maxValue = currValue;
                maxAction = action;//
            }
        }
        return maxValue;
    }

    public HashMap<String, HashMap<String, Double>> QLearning(double eta, double alpha, double gamma, int n){
        HashMap<String, HashMap<String, Double>> q = new HashMap<String, HashMap<String, Double>>();
        CliffState cs = new CliffState(w, l);

        // initialize Q arbitrarily,初始化Q
        //将整个这个地图进行初始化，然后将这个，每一个小格子的动作的概率都均等
        for(int i = 0; i < w; i++){
            for(int j = 0; j < l; j++){
                HashMap<String, Double> qa = new HashMap<String, Double>();
                qa.put("up", 0.25);
                qa.put("down", 0.25);
                qa.put("right", 0.25);
                qa.put("left", 0.25);
                q.put(String.format("(%d, %d)", i, j), qa);
            }
        }
        System.out.println("初始化"+"\n"+q);
        List<Integer> list=new ArrayList<>();//使用该list的目的是为了使用这个统计这个reward的值
        //下面进行这个episode的操作，主要是进行迭代的次数，得出这个价值函数然后进行收敛
        for(int i = 0; i < n; i++){
            cs.reset();
            String s = cs.getState();
            int rs = 0;
            while(!cs.terminate()){
                //使用eta贪心策略进行搜索行为和动作，当获得到这个悬崖的状态的时候，对其进行搜索与探索,q.get(s)的值为这个value值0.5
                String a = etaGreedy(q.get(s), eta);
                System.out.println("the etaGreedy action is:"+a);
                cs.action(a);
                //然后使用用该策略生成的这个动作，进行下一步的这个奖励函数
                int r = cs.getReward();//环境对该智能体做出的动作给的这个奖励
                String s_n = cs.getState();//此时，环境对该动作做出的进行获得其状态
                HashMap<String, Double> qa = q.get(s);
                //然后从这个Q表中找出下一个动作产生的最大的这个状态对应的action
                Double qav = qa.get(a) + alpha*(r + gamma*(getMaxQAV(q.get(s_n))) - qa.get(a));
                qa.put(a, qav);//对动作和收益进行更新
                q.put(s, qa);//然后将整个进行状态进行更改

                ///更新其悬崖的状态，然后对这个奖励进行累加
                s = s_n;
                rs += r;
                 System.out.printf("%d: %s-%s %f\n", i, s, a, qav);
            }
            if(i==(n-1)){
                list.add(rs);
                //System.out.printf("%d\n",rs);
            }else{
                list.add(rs);
                //System.out.printf("%d, ",rs);
            }

        }
        System.out.println("----------------------------");
        System.out.println("更改之后得结果"+"\n"+q);
        System.out.println("——————————————————————Q-learning——————————————————————");
//        for (int i = 0; i < list.size(); i++) {
////            System.out.println(list.get(i));
////        }
        //System.out.println("——————————————————————Q-learning——————————————————————");
        //然后将list写入到文件之中
        try {
            String file="E:\\研究生\\研究生\\论文\\cloudsim学习文档\\cloudsim-5 (2).0\\q-learning.csv";
            File filename= new File(file);
            FileUtils.writeLines(filename,list);
        } catch (Exception e) {
            e.printStackTrace();
        }
        System.out.println("文件保存完成");
        return q;
    }

    public HashMap<String, HashMap<String, Double>> Sarsa(double eta, double alpha, double gamma, int n){
        HashMap<String, HashMap<String, Double>> q = new HashMap<String, HashMap<String, Double>>();
        CliffState cs = new CliffState(w, l);

        // initialize Q arbitrarily
        for(int i = 0; i < w; i++){
            for(int j = 0; j < l; j++){
                HashMap<String, Double> qa = new HashMap<String, Double>();
                qa.put("up", 0.25);
                qa.put("down", 0.25);
                qa.put("right", 0.25);
                qa.put("left", 0.25);
                q.put(String.format("(%d, %d)", i, j), qa);
            }
        }
        List<Integer> list=new ArrayList<>();
        for(int i = 0; i < n; i++){

            cs.reset();
            String s = cs.getState();
            String a = etaGreedy(q.get(s), eta);
            int rs = 0;
            while(!cs.terminate()){
                cs.action(a);
                int r = cs.getReward();
                String s_n = cs.getState();
                String a_n = etaGreedy(q.get(s_n), eta);
                HashMap<String, Double> qa = q.get(s);
                Double qav = qa.get(a) + alpha*(r + gamma*(q.get(s_n).get(a_n)) - qa.get(a));
                qa.put(a, qav);
                q.put(s, qa);
                s = s_n;
                a = a_n;
                rs += r;
                // System.out.printf("%d: %s-%s %f\n", i, s, a, qav);
            }
            if(i==(n-1)){
                list.add(rs);
                //System.out.printf("%d\n",rs);
            }else{
                list.add(rs);
                //System.out.printf("%d, ",rs);
            }
        }
        System.out.println("--------------------------SARSA------------------------");
//        for (int i = 0; i < list.size(); i++) {
//            System.out.println(list.get(i));
//        }
        System.out.println("--------------------------SARSA------------------------");
        try {
            String file="E:\\研究生\\研究生\\论文\\cloudsim学习文档\\cloudsim-5 (2).0\\sarsa.csv";
            File filename= new File(file);
            FileUtils.writeLines(filename,list);
        } catch (Exception e) {
            e.printStackTrace();
        }
        System.out.println("保存完成");
        return q;
    }

    public void printPolicy(HashMap<String, HashMap<String, Double>> q){
        HashMap<String, String> trajectory = new HashMap<String, String>();
        CliffState cs = new CliffState(w,l);
        cs.reset();
        while(!cs.terminate()){
            String s = cs.getState();
            String a = etaGreedy(q.get(s), 0.0);
            trajectory.put(s, a.substring(0,1));
            cs.action(a);
            // System.out.printf("%s-%s", s, a);
        }
        System.out.printf("\n.");
        for(int i = 0; i < w; i++){
            System.out.printf("_");
        }
        System.out.printf(".\n");
        for(int j = (l-1); j > -1; j--){
            System.out.printf("|");
            for(int i = 0; i < w; i++){
                String a = trajectory.get(String.format("(%d, %d)", i,j));
                if(a != null){
                    System.out.printf("%s",a);
                }else{
                    System.out.printf(" ");
                }
            }
            System.out.printf("|\n");
        }
        System.out.printf(".");
        for(int i = 0; i < w; i++){
            System.out.printf("_");
        }
        System.out.printf(".\n");
        // System.out.printf("\n");
    }

    public static void main(String[] args){
         CliffWalking cw = new CliffWalking(12, 4);
         //cw.Sarsa(0.1, 0.01, 1.0, 100);
        // cw.Sarsa(0.1, 0.05, 1.0, 1000);
        // cw.Sarsa(0.1, 0.1, 1.0, 1000);
        // cw.Sarsa(0.1, 0.25, 1.0, 1000);
        // cw.Sarsa(0.1, 0.5, 1.0, 1000);


        // cw.Sarsa(0.01, 0.5, 1.0, 1000);
        // cw.Sarsa(0.1, 0.5, 1.0, 1000);
        // cw.Sarsa(0.25, 0.5, 1.0, 1000);
        // cw.Sarsa(0.5, 0.5, 1.0, 1000);

        // cw.Sarsa(0.01, 0.5, 0.01, 1000);
        // cw.Sarsa(0.01, 0.5, 0.1, 1000);
        // cw.Sarsa(0.01, 0.5, 0.5, 1000);
        // cw.Sarsa(0.01, 0.5, 1.0, 1000);

        // cw.QLearning(0.1, 0.01, 1.0, 1000);
        // cw.QLearning(0.1, 0.05, 1.0, 1000);
        // cw.QLearning(0.1, 0.1, 1.0, 1000);
        // cw.QLearning(0.1, 0.25, 1.0, 1000);
        // cw.QLearning(0.1, 0.5, 1.0, 1000);

       //  cw.QLearning(0.01, 0.5, 1.0, 100);
        // cw.QLearning(0.1, 0.5, 1.0, 1000);
        // cw.QLearning(0.25, 0.5, 1.0, 1000);
        // cw.QLearning(0.5, 0.5, 1.0, 1000);

        // cw.QLearning(0.01, 0.5, 0.01, 1000);
        // cw.QLearning(0.01, 0.5, 0.1, 1000);
        // cw.QLearning(0.01, 0.5, 0.5, 1000);
        // cw.QLearning(0.01, 0.5, 1.0, 1000);

        //CliffWalking cw = new CliffWalking(12, 4);
        // CliffWalking cw = new CliffWalking(4, 6);
        //CliffWalking cw = new CliffWalking(3, 6);
        HashMap<String, HashMap<String, Double>> qS = cw.Sarsa(0.05, 0.5, 1.0, 1000);
        HashMap<String, HashMap<String, Double>> qQL = cw.QLearning(0.05, 0.5, 1.0, 1000);

        cw.printPolicy(qS);
        cw.printPolicy(qQL);
    }
}