function [yearstorms,storms,smoothstorms,tcreconshort,yearreconshort,tcinstr,yearinstr,tcdist,nrecon,nreconshort,resid] = StatModelPaleoTC(nsurr1,ibias);
% (C) Michael Mann, 2008.
%
% Analysis interval:
nyears=137; % 1870-2006
% SSTdatasets
%idata=1 % HadISST
%idata=2 % Kaplan
%idata=3 % ERSST
idata=4; % average of all three products
%
%inino=1 % use NINO3.4 series
inino=2; % use NINO3 series
%
% Read in data
% note that Nino3 and NAO years correspond to "D" in DJF or DJFM)
%
% read in TC counts
load Data/TCUpdate.dat
year=TCUpdate(1:nyears,1);
yr0=year(1);
yrN=year(nyears);
storms=TCUpdate(1:nyears,2);
% read in Jones et al DJFM NAO series, first year is 1870
nao=TCUpdate(1:nyears,5);
% read in SST datasets
load Data/HadISST_MDR_ASO.dat
load Data/HadISST_NINO3p4_DJF.dat
load Data/HadISST_NINO3_DJF.dat
load Data/KAPLAN_MDR_ASO.dat
load Data/KAPLAN_NINO3p4_DJF.dat
load Data/KAPLAN_NINO3_DJF.dat
load Data/ERSST_MDR_ASO.dat
load Data/ERSST_NINO3p4_DJF.dat
load Data/ERSST_NINO3_DJF.dat
mdrh=HadISST_MDR_ASO(1:nyears,2); % Had MDR file missing 1869 value
mdrk=KAPLAN_MDR_ASO(2:nyears+1,2);
mdre=ERSST_MDR_ASO(2:nyears+1,2);
if (inino==1)
    ninoh=HadISST_NINO3p4_DJF(2:nyears+1,2);
    ninok=KAPLAN_NINO3p4_DJF(2:nyears+1,2);
    ninoe=ERSST_NINO3p4_DJF(2:nyears+1,2);
else
    ninoh=HadISST_NINO3_DJF(2:nyears+1,2);
    ninok=KAPLAN_NINO3_DJF(2:nyears+1,2);
    ninoe=ERSST_NINO3_DJF(2:nyears+1,2);
end
%
% calculate correlations between different products
cninohk=lincor(ninoh,ninok);
cninohe=lincor(ninoh,ninoe);
cninoke=lincor(ninok,ninoe);
cmdrhk=lincor(mdrh,mdrk);
cmdrhe=lincor(mdrh,mdre);
cmdrke=lincor(mdrk,mdre);
mdrhsmooth=lowpassmin(mdrh,0.1);
mdrksmooth=lowpassmin(mdrk,0.1);
mdresmooth=lowpassmin(mdre,0.1);
cmdrhksmooth=lincor(mdrhsmooth,mdrksmooth);
cmdrhesmooth=lincor(mdrhsmooth,mdresmooth);
cmdrkesmooth=lincor(mdrksmooth,mdresmooth);
%
% define MDR and NINO series
if (idata==1) 
    mdr=mdrh;
    nino=ninoh;
else
    if (idata==2)
        mdr=mdrk;
        nino=ninok;
    else
        if (idata==3)
           mdr=mdre;
           nino=ninoe;  
        else
            mdr=0.3333*(mdrh+mdrk+mdre);
            nino=0.3333*(ninoh+ninok+ninoe);
        end
    end
end
%
% store values for regression
mdrs=mdr(1:nyears);
ninos=nino(1:nyears);
naos=nao(1:nyears);
%
% set values for 2007 prediction 
year(nyears+1)=2007;
mdr(nyears+1)=mdr(135); % about same as '05 Temps
nino(nyears+1)=-0.2; % weak La Nina
nao(nyears+1)=0.47; % climatological mean NAO
% determine correlations between TC numbers and each of the three state
% variables, and determine dofs taking into account serial correlation
rho0 = lincor(storms(1:nyears),storms(1:nyears),1);
rho1 = lincor(mdrs,mdrs,1);
rho2 = lincor(ninos,ninos,1);
rho3 = lincor(naos,naos,1);
corrmdr=lincor(storms(1:nyears),mdrs);
nprimemdr = nyears*(1-rho0*rho1)/(1+rho0*rho1);
corrnino=lincor(storms(1:nyears),ninos);
nprimenino = nyears*(1-rho0*rho2)/(1+rho0*rho2);
corrnao=lincor(storms(1:nyears),naos);
nprimenao = nyears*(1-rho0*rho3)/(1+rho0*rho3);
%
Y = storms(1:nyears);
wons=ones(nyears,1);
%nsplit = 69;  % 1870-1938/1939-2006 split validation
nsplit = 74;  % 1870-1943/1944-2006 split validation
Yl=Y(1:nsplit);
Ye=Y(nsplit+1:nyears);
%
%
if (ibias==1) % increase pre-1944 TC count by 1.2 storm per year as per Mann et al GRL '07
diff=1.2;
Yl=Yl+diff;
Y(1:nsplit)=Yl;
Y(nsplit+1:nyears)=Ye;
storms=Y;
end
if (ibias==2) % impose "Landsea '07" adjustments as described in Mann et al Eos '07
for i=1:96
    tcs(i)=storms(i)+3.2;
end
for i=1:15
    tcs(i)=storms(i)+5;
end
for i=97:133
    tcs(i)=storms(i)+1;
end
for i=134:137
    tcs(i)=storms(i);
end
storms=tcs';
Y = storms(1:nyears);
storms=Y;
end
%
% Poisson regression using MDR, NINO3 and NAO as predictors
%
X = [mdrs ninos naos wons];
[BETA4,MU4,DEV,DF,SE]=poisson(Y,X);
%
%
% adequacy of the poisson regression model
%
residdev=DEV
residdof=DF
v = residdof; % degrees of freedom
chisqr=residdev;
chisqrred = chisqr/v; % reduced chi-squared
alpha = 1-gammainc(chisqr/2,v/2) % probability of type 1 error
%
% adequacy of the null (constant) model
%
X0 = [wons];
[BETA0,MU0,DEV0,DF0,SE]=poisson(Y,X0);
nulldev=DEV0
nulldof=DF0
v = nulldof; % degrees of freedom
chisqr=nulldev;
chisqrred = chisqr/v; % reduced chi-squared
alpha0 = 1-gammainc(chisqr/2,v/2) % probability of type 1 error
% 
% likelihood test for significance of model
%
devdiff = DEV0-DEV
dof=3
v = dof; % degrees of freedom
chisqr=devdiff;
chisqrred = chisqr/v; % reduced chi-squared
alphadiff = 1-gammainc(chisqr/2,v/2) % probability of type 1 error
%
%
ymodel4=exp(BETA4(1)*X(:,1)+BETA4(2)*X(:,2)+BETA4(3)*X(:,3)+BETA4(4));
resid4=round(mean(Y)+Y-ymodel4);
% prediction of 2007 value
tcpredict4(1)=exp(BETA4(1)*mdr(nyears+1)+BETA4(2)*nino(nyears+1)+BETA4(3)*nao(nyears+1)+BETA4(4));
% late validation/early calibration
Xl = [mdrs(1:nsplit) ninos(1:nsplit) naos(1:nsplit) wons(1:nsplit)];
[BETA4l,MU4l,DEV,DF,SE]=poisson(Yl,Xl);
%    determine residual over full period
ymodel4l=exp(BETA4l(1)*X(:,1)+BETA4l(2)*X(:,2)+BETA4l(3)*X(:,3)+BETA4l(4));
resid4l=round(mean(Y)+Y-ymodel4l);
%    calculate calibration and validation R2 and MSE
rsq4lc=lincor(ymodel4l(1:nsplit),Y(1:nsplit))^2;
MSE4lc=std(resid4l(1:nsplit))^2;
rsq4lv=lincor(ymodel4l(nsplit+1:nyears),Y(nsplit+1:nyears))^2;
MSE4lv=std(resid4l(nsplit+1:nyears))^2;
% late calibration/early validation
Xe = [mdrs(nsplit+1:nyears) ninos(nsplit+1:nyears) naos(nsplit+1:nyears) wons(nsplit+1:nyears)];
[BETA4e,MU4e,DEV,DF,SE]=poisson(Ye,Xe);
%    determine residual over full period
ymodel4e=exp(BETA4e(1)*X(:,1)+BETA4e(2)*X(:,2)+BETA4e(3)*X(:,3)+BETA4e(4));
resid4e=round(mean(Y)+Y-ymodel4e);
%    calculate calibration and validation R2 and MSE
rsq4ec=lincor(ymodel4e(nsplit+1:nyears),Y(nsplit+1:nyears))^2;
MSE4ec=std(resid4e(nsplit+1:nyears))^2;
rsq4ev=lincor(ymodel4e(1:nsplit),Y(1:nsplit))^2;
MSE4ev=std(resid4e(1:nsplit))^2;
% full regression stats and average early and late calibration/validation stats
MSE4r=std(resid4)^2;
MSE4c=0.5*(MSE4ec+MSE4lc);
MSE4v=0.5*(MSE4ev+MSE4lv);
rsq4r=lincor(ymodel4,Y)^2;
rsq4c=0.5*(rsq4ec+rsq4lc);
rsq4v=0.5*(rsq4ev+rsq4lv);
% define full validation residual series
resid4(1:nsplit)=resid4e(1:nsplit);
resid4(nsplit+1:nyears)=resid4l(nsplit+1:nyears);
resid=resid4;
% compute climatology and persistence MSE and R2
climat=round(mean(storms(1:nyears)));
resid5=climat-storms(1:nyears);
MSEclim=std(resid5)^2;
persist(1:nyears-1)=storms(1:nyears-1);
persist=persist';
resid6=persist-storms(2:nyears);
MSEpersist=std(resid6)^2;
rsqpersist=lincor(persist,storms(2:nyears))^2;
%
% get decadal calibration and split calibration/validation results and residuals
%
freq=0.1;
years=year(1:nyears);
smoothstorms=lowpassadaptive(storms(1:nyears),freq);
% calculate smoothed full calibration fit
smoothMU4=lowpassadaptive(MU4,freq);
smoothMU4e=lowpassmin(ymodel4e(1:nsplit),freq);
smoothMU4l=lowpassmin(ymodel4l(nsplit+1:nyears),freq);
% calculate decadally-smoothed split calib/valid fits
smoothresid4e=lowpassmin(resid4e,freq);
smoothresid4l=lowpassmin(resid4l,freq);
% calculate prediction uncertainties from calibration residuals in split calib/valid 
% experiments
resid4early=smoothresid4e(nsplit+1:nyears);
resid4late=smoothresid4l(1:nsplit);
unc4e=std(resid4early);
unc4l=std(resid4late);
%
% define standard error in decadal statistical model as mean of standard
% deviation of early and late validation experiments
sderrdec = 0.5*(unc4e+unc4l);
%
earlymeanactual(1:nsplit)=mean(storms(1:nsplit));
earlymeanpredict(1:nsplit)=mean(ymodel4e(1:nsplit));
latemeanactual(1:nyears-nsplit)=mean(storms(nsplit+1:nyears));
latemeanpredict(1:nyears-nsplit)=mean(ymodel4l(nsplit+1:nyears));
tearly=(mean(earlymeanactual)-mean(earlymeanpredict))/sqrt(mean(earlymeanactual)/length(earlymeanactual));
tlate=(mean(latemeanactual)-mean(latemeanpredict))/sqrt(mean(latemeanactual)/length(latemeanactual));
ymodel=ymodel4;
BETA=BETA4;
BETAe=BETA4e;
BETAl=BETA4l;
%
% read in decadally-smoothed paleo Nino3 and MDR reconstructions
load Data/nino3valid
yearlong=nino3valid(:,1);
nino3long=nino3valid;
ninolong = nino3valid(:,3);
load Data/tamdrvalid
tamdrlong=tamdrvalid;
mdrlong = tamdrvalid(:,3);
% read in luterbacher DJF NAO reconstruction
% (note that DJF NAO for 1500 by our convention is actually the 1501 DJF
% Luterbacher value, so stagger by one)
load Data/nao_seadat.txt
nsea=floor(length(nao_seadat)/4);
for i=1:nsea
    iindex=i*4+1;
    yearnao(i)=nao_seadat(iindex-4,1);
    naolon(i)=nao_seadat(iindex,2);
end
load Data/nao_mon.dat
nmon=floor(length(nao_mon)/12);
for i=1:nmon
    iindex=i*12;
    yearnao(nsea+i)=nao_mon(iindex,1);
    naolon(nsea+i)=0.33333*sum(nao_mon(iindex:iindex+2,2));
end
naolon=naolon';
naolong=lowpassmin(naolon,freq);
% scale series so they have same decadal mean and variance as their
% instrumental counterparts (1870-recent)
ninoverlap=ninolong(1871:2007);
mdroverlap=mdrlong(1871:2007);
naooverlap=naolong(370:500);
mdrsmooth=lowpassmin(mdrs,freq);
ninosmooth=lowpassmin(ninos,freq);
naosmooth=lowpassmin(naos(1:131),freq);
%
ninolong=ninolong*std(ninosmooth)/std(ninoverlap);
ninolong=ninolong+mean(ninosmooth)-mean(ninoverlap);
mdrlong=mdrlong*std(mdrsmooth)/std(mdroverlap);
mdrlong=mdrlong+mean(mdrsmooth)-mean(mdroverlap);
naolong=naolong*std(naosmooth)/std(naooverlap);
naolong=naolong+mean(naosmooth)-mean(naooverlap);
%
nrecon=2007-501+1;
yearrecon=yearlong(501:2007);
% assign NAO the mean calibration period value prior to availability of NAO reconstruction
naorecon(1:nrecon)=mean(naos);
naorecon(1000:1500)=naolong;
naorecon=naorecon';
mdrrecon=mdrlong(501:2007);
ninorecon=ninolong(501:2007);
% define standard errors
ninostd = (nino3long(501:2007,5)-nino3long(501:2007,4))/4;
mdrstd = (tamdrlong(501:2007,5)-tamdrlong(501:2007,4))/4;
% define standard errors for NAO recon based on Luterbacher RE scores
naostd(1:nrecon)=0.0;
RE(1000:1155)=0.62;
RE(1156:1321)=0.44;
RE(1322:1362)=0.7;
RE(1363:1500)=0.9;
naostd(1000:1500)=std(naosmooth)*sqrt(1-RE(1000:1500));
%
%
% now apply regression model to proxy climate index reconstructions to yield TC reconstruction back to AD 500
%
tcrecon=exp(BETA4(1)*mdrrecon+BETA4(2)*ninorecon+BETA4(3)*naorecon+BETA4(4));
%
% generate monte carlo surrogates for predictors
stderrormdr = 0.15;
stderrornino = 0.25;
nrecon=length(tcrecon);
% generate nsurr1 monte carlo realizations and determine two sigma limits
% (i.e. lower and upper 2.5% corresponding to +/- 2 sigma)
for icount=1:nsurr1
for i=1:nrecon
   mdrsurr(i)=mdrrecon(i)+mdrstd(i)*gasdev;
   ninosurr(i)=ninorecon(i)+ninostd(i)*gasdev;
   naosurr(i)=naorecon(i)+naostd(i)*gasdev;
end
tcsurr=exp(BETA4(1)*mdrsurr'+BETA4(2)*ninosurr'+BETA4(3)*naorecon+BETA4(4))+sderrdec*gasdev;
tcdist(:,icount)=tcsurr;
end
nreconshort=1850-500+1;
yearreconshort=yearrecon(1:nreconshort);
tcreconshort=tcrecon(1:nreconshort);
yearinstr=(1850:2006);
tcinstr(1:20)=tcrecon(nreconshort+1:nreconshort+20);
tcinstr(21:157)=smoothMU4;
yearinstr=yearinstr';
yearstorms=yearinstr(21:157);
tcinstr=tcinstr';
end



    








