% Simulate temperature record exceedance statistics using semi-empirical approach
% (C) Michael Mann, 2015.
%
% set parameters of simulation
nyearsobs=165; % length of 1850-2014 instrumental record
nyearsGISTEMP=135;
% iglobal=0; % NH=0 / global=1;
iglobal=1;
itemp=0; % GISTEMP=0 / HadCRUT4=1 
imodel=2; % CMIP5-TAS=1 / CMIP5-TASST=2
iboot=0; % assume Gaussian innovations=0 / estimate innovations via bootstrap=1
iaer = 0; % use only CMIP5 models that have both 1st & 2nd aer indirect effects
icorrect = 1; % update forced response from '86-'13 based on Schmidt et al 2013
nensemble=1000; % ensemble size for Monte Carlo simulations
%
% read in instrumental record (NH mean temp)
%
if (itemp==1) % use HadCRUT4
    if (iglobal==0)
       load Data/HadCRUT4_nh_annual.dat
%      only use 1880-2014 interval for consistency
       nhHadCRU=HadCRUT4_nh_annual(31:165,2);
       TObserv=nhHadCRU;
    else
       load Data/HadCRUT4_globe_annual.dat
%      only use 1880-2014 interval for consistency
       globeHadCRU=HadCRUT4_globe_annual(31:165,2);
       TObserv=globeHadCRU;
    end      
else % use GISTEMP 1880-2014
    if (iglobal==0)
       load Data/GISTEMP_NH.dat
       nhGISTEMP=GISTEMP_NH(:,2);
       TObserv=nhGISTEMP;
    else
       load Data/GISTEMP_Globe.dat
       globeGISTEMP=GISTEMP_Globe(:,2);
       TObserv=globeGISTEMP;
    end
end
%
%
%  first determine anthropogenic-only component
%
%
if (imodel==1) % use CMIP5 mean SAT
   if (iglobal==0)
      if (iaer)
       load Data/NHtas_anthro_AIE.dat % subset of 1st+2nd indirect
       yearmodel=NHtas_anthro_AIE(:,1); % subset of 1st+2nd indirect
       tempmodel=NHtas_anthro_AIE(:,2); % subset of 1st+2nd indirect
      else
       load Data/NHtas_anthro.dat
       yearmodel=NHtas_anthro(:,1);
       tempmodel=NHtas_anthro(:,2);
      end
   else
      if (iaer)
       load Data/GLtas_anthro_AIE.dat % use subset of 1st+2nd indirect
       yearmodel=GLtas_anthro_AIE(:,1); % subset of 1st+2nd indirect
       tempmodel=GLtas_anthro_AIE(:,2); % subset of 1st+2nd indirect
      else
       load Data/GLtas_anthro.dat
       yearmodel=GLtas_anthro(:,1);
       tempmodel=GLtas_anthro(:,2);
      end
   end
else
   if (iglobal==0)
      if (iaer)
       load Data/NHtastos_anthro_AIE.dat % use subset of 1st+2nd indirect
       yearmodel=NHtastos_anthro_AIE(:,1); % subset of 1st+2nd indirect
       tempmodel=NHtastos_anthro_AIE(:,2); % subset of 1st+2nd indirect
      else
       load Data/NHtastos_anthro.dat
       yearmodel=NHtastos_anthro(:,1);
       tempmodel=NHtastos_anthro(:,2);
      end
   else
     if (iaer)
       load Data/GLtastos_anthro_AIE.dat % use subset of 1st+2nd indirect
       yearmodel=GLtastos_anthro_AIE(:,1); % subset of 1st+2nd indirect
       tempmodel=GLtastos_anthro_AIE(:,2); % subset of 1st+2nd indirect
      else
       load Data/GLtastos_anthro.dat
       yearmodel=GLtastos_anthro(:,1);
       tempmodel=GLtastos_anthro(:,2);
      end  
   end
end
% smooth modelmodel mean to eliminate interannual noise
frequency = 0.025;
[tempsmoothed,w0,w1,w2,msebest] = lowpassadaptive(tempmodel,frequency);
% extrapolate model simulations to 2014 based on persistence of 20 year trend
nyearsmodel=length(yearmodel); 
series=tempsmoothed(nyearsmodel-19:nyearsmodel);
x1=(1:20)';
x2=(21:29)';    
P1 = polyfit(x1,series,1);
extend=P1(2)+P1(1)*x2;
tempsmoothed(nyearsmodel+1:nyearsmodel+9)=extend;
model=tempsmoothed(20:154); % 1880-2014
if (icorrect) % correct 1986-2013 forced CMIP5 mean response
   load Data/SchmidtCorrectAnthro.dat
   adjust(1:106)=0.0;
   adjust(107:134)=SchmidtCorrectAnthro(:,2);
   adjust(135)=adjust(134); % extend through 2014 by persistence
   model=model+adjust';
end
modelanthro=model-mean(model);
%
%  now determine total forced component
%
if (imodel==1) % use CMIP5 mean SAT
   if (iglobal==0)
      if (iaer)
       load Data/NHtas_AIE.dat % subset of 1st+2nd indirect
       yearmodel=NHtas_AIE(:,1); % subset of 1st+2nd indirect
       tempmodel=NHtas_AIE(:,2); % subset of 1st+2nd indirect
      else
       load Data/NHtas_all.dat
       yearmodel=NHtas_all(:,1);
       tempmodel=NHtas_all(:,2);
      end
   else
      if (iaer)
       load Data/GLtas_AIE.dat % use subset of 1st+2nd indirect
       yearmodel=GLtas_AIE(:,1); % subset of 1st+2nd indirect
       tempmodel=GLtas_AIE(:,2); % subset of 1st+2nd indirect
      else
       load Data/GLtas_all.dat
       yearmodel=GLtas_all(:,1);
       tempmodel=GLtas_all(:,2);
      end
   end
else
   if (iglobal==0)
      if (iaer)
       load Data/NHtastos_AIE.dat % use subset of 1st+2nd indirect
       yearmodel=NHtastos_AIE(:,1); % subset of 1st+2nd indirect
       tempmodel=NHtastos_AIE(:,2); % subset of 1st+2nd indirect
      else
       load Data/NHtastos_all.dat
       yearmodel=NHtastos_all(:,1);
       tempmodel=NHtastos_all(:,2);
      end
   else
     if (iaer)
       load Data/GLtastos_AIE.dat % use subset of 1st+2nd indirect
       yearmodel=GLtastos_AIE(:,1); % subset of 1st+2nd indirect
       tempmodel=GLtastos_AIE(:,2); % subset of 1st+2nd indirect
      else
       load Data/GLtastos_all.dat
       yearmodel=GLtastos_all(:,1);
       tempmodel=GLtastos_all(:,2);
      end  
   end
end
% extrapolate model simulations to 2014 based on persistence of 20 year trend
nyearsmodel=length(yearmodel); 
series=tempmodel(nyearsmodel-19:nyearsmodel);
x1=(1:20)';
x2=(21:29)';    
P1 = polyfit(x1,series,1);
extend=P1(2)+P1(1)*x2;
tempmodel(nyearsmodel+1:nyearsmodel+9)=extend;
model=tempmodel(20:154); % 1880-2014
if (icorrect) % correct 1986-2013 forced CMIP5 mean response
   load Data/SchmidtCorrectAnthro.dat
   adjust1(1:106)=0.0;
   adjust1(107:134)=SchmidtCorrectAnthro(:,2);
   adjust1(135)=adjust1(134); % extend through 2014 by persistence
   load Data/SchmidtCorrectNatural.dat
   adjust2(1:106)=0.0;
   adjust2(107:134)=SchmidtCorrectNatural(:,2);
   adjust2(135)=adjust2(134); % extend through 2014 by persistence
   model=model+adjust1'+adjust2';
end
year=(1880:2014)';
nyears=135;
nyearsfit=135; % use full 1880-2014 data in parameter estimation
%nyearsfit=126; % only use 1880-2005 data in parameter estimation
%nyearsfit=120; % only use 1880-1999 data in parameter estimation
instr=TObserv;
model=model-mean(model);
modelnatural=model-modelanthro;
instr=instr-mean(instr);
resid=instr-model;
residfit=resid(1:nyearsfit);
instrfit=instr(1:nyearsfit);
%
% Fig 1 - plot modeled vs. instrumental NH mean temperatures
%
figure(1)
hold
plot(year,instr,'k');
plot(year,model)
%
figure(2)
hold
plot(year,resid)
natural=resid+modelnatural;
plot(year,natural,'k');
%
%
% examine 3 different null hypotheses for the "natural" (i.e. residual) series
% (1) AR(1) red noise [y_t=rho * y_(t-1)+eps_t]
% (2) best fit ARMA(p,q) process
% (3) persistent AR(1) [y_t=rho_2 *y_(t-1)+eps_t where rho2 taken from full series]
%
% estimate variance and autocorrelation of observational/forced model residual
%
%
rho=lincor(residfit,residfit,1); % actual series autocorrelation
rho2=lincor(instrfit,instrfit,1); % random walk autocorrelation
stochamp=std(residfit);
stochamp2=std(instrfit);
ampfactor=1.0/sqrt(1.0/(1.0-rho^2));
ampfactor2=1.0/sqrt(1.0/(1.0-rho2^2));
ampnoise=stochamp*ampfactor;
ampnoise2=stochamp2*ampfactor2;
innov=(1.0/ampnoise)*(resid(2:nyears)-rho*(resid(1:nyears-1)));
innov2=(1.0/ampnoise2)*(instr(2:nyears)-rho2*(instr(1:nyears-1)));
%
% generate ensemble of surrogate natural variability NH series 
%
% consider AR(1) red noise and FBM models of time series
for j=1:nensemble
    % assign 1st value in series
    if (iboot) % generate a bootstrap surrogate
        iloc=round(1.0+(nyears-2)*rand);
        noise(1,j)=innov(iloc);
        noise2(1,j)=innov2(iloc);
    else % assume gaussian 
        noise(1,j)=gasdev;
        noise2(1,j)=gasdev;
    end
    rednoise(1,j)=noise(1,j);
    randomwalk(1,j)=noise2(1,j);
    % generate remaining series
    for i=2:nyears
        if (iboot) % generate a bootstrap surrogate
          iloc=round(1.0+(nyears-2)*rand);
          noise(i,j)=innov(iloc);
          noise2(i,j)=innov2(iloc);
        else % assume gaussian
          noise(i,j)=gasdev;
          noise2(i,j)=gasdev;
        end
        rednoise(i,j)=rho*rednoise(i-1,j)+noise(i,j);
        randomwalk(i,j)=rho2*randomwalk(i-1,j)+noise2(i,j);
    end
rednoise(:,j)=modelnatural+ampnoise*rednoise(:,j);
randomwalk(:,j)=modelnatural+ampnoise2*randomwalk(:,j);
%fbmsurr = wfbm(H,nyears);
%firstdiffsurr=(fbmsurr(2:nyears)-fbmsurr(1:nyears-1));
%stdfirstdiffsur=std(firstdiffsurr);
%fbmnoise(:,j)=ampnoise*(stdfirstdiff/stdfirstdiffsur)*fbmsurr;
end
%
% consider ARMA(p,q) model of time series
%
Y=residfit;
% consider values p,q=1,2,3,4
LOGL = zeros(4,4); %Initialize
PQ = zeros(4,4);
for p = 0:3
    for q = 0:3
        mod = arima(p,0,q);
        [fit,~,logL] = estimate(mod,Y,'print',false);
        LOGL(p+1,q+1) = logL;
        PQ(p+1,q+1) = p+q;
    end
end
% apply BIC to select order p,q
LOGL = reshape(LOGL,16,1);
PQ = reshape(PQ,16,1);
[~,bic] = aicbic(LOGL,PQ+1,nyearsfit);
mat=reshape(bic,4,4);
minbic=1.0e+8;
for p=0:3
    for q=0:3
        if (mat(p+1,q+1)<=minbic)
            pbest=p;
            qbest=q;
            minbic=mat(p+1,q+1);
            Mdl=arima(p,0,q);
        end
    end
end
% estimate the parameters of the optimal ARMA(p,q) model
EstMdl = estimate(Mdl,Y);
%
% estimate model residuals, check autocorrelation
[E,V,logL] = infer(EstMdl,Y);
% two-sided signif levels for N=135 length series
signif(1)=0.169;
signif(2)=0.1697;
signif(3)=0.1703;
signif(4)=0.171;
signif(5)=0.1716;
signif(6)=0.1723;
signif(7)=0.173;
signif(8)=0.1736;
signif(9)=0.1743;
signif(10)=0.175;
signif(11)=0.1757;
signif(12)=0.1764;
signif(13)=0.1771;
signif(14)=0.1778;
signif(15)=0.1786;
signif(16)=0.1793;
signif(17)=0.1801;
signif(18)=0.1809;
signif(19)=0.18165;
signif(20)=0.1824;
for p=1:20
    lag(p)=p;
    autoc(p)=lincor(E,E,p);
end
figure(7)
hold
plot(lag,autoc);
plot(lag,signif,'r--');
plot(lag,-signif,'r--');
%
% generate ensemble of ARMA(p,q) models of length nyears
for j=1:nensemble
    arma=simulate(EstMdl,nyears);
    armanoise(:,j)=arma+modelnatural;
end
%          
%
% plot first 4 ensemble red noise series along w/ resid
%
figure(3);
hold
plot(year,rednoise(:,1),'r'); 
plot(year,rednoise(:,2),'y'); 
plot(year,rednoise(:,3),'g'); 
plot(year,rednoise(:,4),'m'); 
plot(year,rednoise(:,5),'cyan'); 
plot(year,natural,'k');
plot(year,instr,'k--');
%
% plot first 4 ensemble ARMA noise series along w/ resid
%
figure(4);
hold
plot(year,armanoise(:,1),'r'); 
plot(year,armanoise(:,2),'y'); 
plot(year,armanoise(:,3),'g'); 
plot(year,armanoise(:,4),'m'); 
plot(year,armanoise(:,5),'cyan'); 
plot(year,natural,'k');
plot(year,instr,'k--');
%
% plot first 4 ensemble random walk series along w/ resid
%
figure(5);
hold
plot(year,randomwalk(:,1),'r'); 
plot(year,randomwalk(:,2),'y'); 
plot(year,randomwalk(:,3),'g'); 
plot(year,randomwalk(:,4),'m'); 
plot(year,randomwalk(:,5),'cyan'); 
plot(year,instr,'k--');
%
% now construct full (anthro+natural) series by adding EBM simulation
% and ARMA series
%
for j=1:nensemble
    surftemp(:,j)=model+armanoise(:,j)-modelnatural;
end
% plot first 4 full series series along w/ instrumental series
%
figure(6);
hold
plot(year,surftemp(:,1),'r'); 
plot(year,surftemp(:,2),'y'); 
plot(year,surftemp(:,3),'g'); 
plot(year,surftemp(:,4),'m'); 
plot(year,surftemp(:,5),'cyan'); 
plot(year,instr,'k');
%
% now calculate frequencies for particular record temperature values
% for both natural and anthro+natural simulated NH mean series
%
% consider individual record years: 1998, 2005, 2010, 2014
%
iyear1=119; % 1998
target1=instr(119);
iyear2=126;  % 2005
target2=instr(126);
iyear3=131;  % 2010
target3=instr(131);
iyear4=135;  % 2014
target4=instr(135);
%
% Also look at collective exceedance probabilities, i.e.
% probability that 
% (a) 9 of the 10 warmest years occur since 2000
% and
% (b) 13 of the 15 warmest years occur since 2000
%
% first, rank the instrumental series
%
[instrsort,isortinstr]=sort(instr,'descend');
isum15=0;
isum10=0;
ithresh=121; % year 2000 corresponds to year #121
for i=1:10
   isum10=isum10+(isortinstr(i)>=ithresh);
end 
for i=1:15
   isum15=isum15+(isortinstr(i)>=ithresh);
end
'observ # of 10 warmest since 2000',isum10
'observ # of 15 warmest since 2000',isum15
%
fcount1=0;
ncount1=0;
acount1=0;
rcount1=0;
fcount2=0;
ncount2=0;
acount2=0;
rcount2=0;
fcount3=0;
ncount3=0;
acount3=0;
rcount3=0;
fcount4=0;
ncount4=0;
acount4=0;
rcount4=0;
for j=1:nensemble;
    [fsort,isortf]=sort(surftemp(:,j),'descend');
    [nsort,isortn]=sort(rednoise(:,j),'descend');
    [asort,isorta]=sort(armanoise(:,j),'descend');
    [rsort,isortr]=sort(randomwalk(:,j),'descend');
    fsum10(j)=0;
    nsum10(j)=0;
    asum10(j)=0;
    rsum10(j)=0;
    fsum15(j)=0;
    nsum15(j)=0;
    asum15(j)=0;
    rsum15(j)=0;
    for i=1:10
       fsum10(j)=fsum10(j)+(isortf(i)>=ithresh);
       nsum10(j)=nsum10(j)+(isortn(i)>=ithresh);
       asum10(j)=asum10(j)+(isorta(i)>=ithresh);
       rsum10(j)=rsum10(j)+(isortr(i)>=ithresh);
    end 
    for i=1:15
       fsum15(j)=fsum15(j)+(isortf(i)>=ithresh);
       nsum15(j)=nsum15(j)+(isortn(i)>=ithresh);
       asum15(j)=asum15(j)+(isorta(i)>=ithresh);
       rsum15(j)=rsum15(j)+(isortr(i)>=ithresh);
    end
    fcount1=fcount1+(surftemp(iyear1,j)>=target1);
    fcount2=fcount2+(surftemp(iyear2,j)>=target2);
    fcount3=fcount3+(surftemp(iyear3,j)>=target3);
    fcount4=fcount4+(surftemp(iyear4,j)>=target4);
    ncount1=ncount1+(rednoise(iyear1,j)>=target1);
    ncount2=ncount2+(rednoise(iyear2,j)>=target2);
    ncount3=ncount3+(rednoise(iyear3,j)>=target3);
    ncount4=ncount4+(rednoise(iyear4,j)>=target4);
    acount1=acount1+(armanoise(iyear1,j)>=target1);
    acount2=acount2+(armanoise(iyear2,j)>=target2);
    acount3=acount3+(armanoise(iyear3,j)>=target3);
    acount4=acount4+(armanoise(iyear4,j)>=target4);
    rcount1=rcount1+(randomwalk(iyear1,j)>=target1);
    rcount2=rcount2+(randomwalk(iyear2,j)>=target2);
    rcount3=rcount3+(randomwalk(iyear3,j)>=target3);
    rcount4=rcount4+(randomwalk(iyear4,j)>=target4);
end
ffrac1=100*fcount1/nensemble;
ffrac2=100*fcount2/nensemble;
ffrac3=100*fcount3/nensemble;
ffrac4=100*fcount4/nensemble;
nfrac1=100*ncount1/nensemble;
nfrac2=100*ncount2/nensemble;
nfrac3=100*ncount3/nensemble;
nfrac4=100*ncount4/nensemble;
afrac1=100*acount1/nensemble;
afrac2=100*acount2/nensemble;
afrac3=100*acount3/nensemble;
afrac4=100*acount4/nensemble;
rfrac1=100*rcount1/nensemble;
rfrac2=100*rcount2/nensemble;
rfrac3=100*rcount3/nensemble;
rfrac4=100*rcount4/nensemble;
'% exceedance 1998 2005 2010 2014 (anthro)',ffrac1,ffrac2,ffrac3,ffrac4
'% exceedance 1998 2005 2010 2014 (natural-red)',nfrac1,nfrac2,nfrac3,nfrac4
'% exceedance 1998 2005 2010 2014 (natural-ARMA)',afrac1,afrac2,afrac3,afrac4
'% exceedance 1998 2005 2010 2014 (natural-randwalk)',rfrac1,rfrac2,rfrac3,rfrac4
ffrac10=100*sum(fsum10>=9)/nensemble;
nfrac10=100*sum(nsum10>=9)/nensemble;
afrac10=100*sum(asum10>=9)/nensemble;
rfrac10=100*sum(rsum10>=9)/nensemble;
'probability at least 9 of 10 warmest since 2000 (anthro)',ffrac10
'probability at least 9 of 10 warmest since 2000 (natural-red)',nfrac10
'probability at least 9 of 10 warmest since 2000 (natural-ARMA)',afrac10
'probability at least 9 of 10 warmest since 2000 (natural-randwalk)',rfrac10
ffrac15=100*sum(fsum15>=13)/nensemble;
nfrac15=100*sum(nsum15>=13)/nensemble;
afrac15=100*sum(asum15>=13)/nensemble;
rfrac15=100*sum(rsum15>=13)/nensemble;
'probability at least 13 of 15 warmest since 2000 (anthro)',ffrac15
'probability at least 13 of 15 warmest since 2000 (natural-red)',nfrac15
'probability at least 13 of 15 warmest since 2000 (natural-ARMA)',afrac15
'probability at least 13 of 15 warmest since 2000 (natural-randwalk)',rfrac15
%
% 
%
%
%
% THE END
%
%