Skip to content

Commit 3ce910a

Browse files
author
Sebastian Lehmann
committed
first commit
1 parent 2dd8bd2 commit 3ce910a

35 files changed

+3393
-0
lines changed

‎profile.txt‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
memhash=25
2+
maxorder=8
3+
4+
counter_limit=100
5+
6+
match_model=0
7+
match_win_size=25
8+
match_model_rate=0.008
9+
10+
word_model=0
11+
word_model_mem=24
12+
word_model_limit=100
13+
14+
sparse_model=0
15+
sparse_model_mem=23
16+
sparse_model_limit=80
17+
18+
record_model=0
19+
record_model_mem=22
20+
record_model_limit=80
21+
22+
indirect_model=0
23+
indirect_model_mem=22
24+
indirect_model_limit=80
25+
26+
27+
final_mix_rate=0.006
28+
final_chain_rate=0.005
29+
30+
final_sse=1
31+
final_sse_limit=150
32+
final_sse_mix_rate=0.005

‎src/codec/cm.cpp‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
#include "cm.h"
2+
3+
4+
CM::CM(BaseRangeCoder *RCoder,TCMParam *CMParam,bool verbose)
5+
:Encoder(RCoder),Param(CMParam)
6+
,myPPM(Param->maxorder,Param->memhash,Param->counter_limit),
7+
myMixer(Param->final_mix_rate,Param->final_chain_rate)
8+
{
9+
maxorder = Param->maxorder;
10+
11+
myMatchModel=NULL;
12+
myWordModel=NULL;
13+
mySparseModel=NULL;
14+
myRecordModel=NULL;
15+
myIndirectModel=NULL;
16+
mySSEModel=NULL;
17+
18+
if (Param->match_model) myMatchModel = new MatchModel(Param->match_win_size,Param->match_win_size-2,Param->match_model_rate);
19+
if (Param->word_model) myWordModel = new WordModel(Param->word_model_mem,Param->word_model_limit);
20+
if (Param->sparse_model) mySparseModel = new SparseModel(Param->sparse_model_mem,Param->sparse_model_limit);
21+
if (Param->record_model) myRecordModel = new RecordModel(Param->record_model_mem,Param->record_model_limit);
22+
if (Param->indirect_model) myIndirectModel = new IndirectModel(Param->indirect_model_mem,Param->indirect_model_limit);
23+
if (Param->final_sse) mySSEModel = new SSEModel();
24+
25+
if (verbose)
26+
{
27+
printf("cm: %i mb",myPPM.GetMemSize()>>20);
28+
if (myWordModel) printf(" wm: %i mb",myWordModel->GetMemSize()>>20);
29+
if (myMatchModel) printf(" mm: %i mb",myMatchModel->GetMemSize()>>20);
30+
if (mySparseModel) printf(" sp: %i mb",mySparseModel->GetMemSize()>>20);
31+
if (myRecordModel) printf(" rc: %i mb",myRecordModel->GetMemSize()>>20);
32+
if (myIndirectModel) printf(" id: %i mb",myIndirectModel->GetMemSize()>>20);
33+
if (mySSEModel) printf(" (sse: %i mb)",mySSEModel->GetMemSize()>>20);
34+
printf("\n");
35+
}
36+
l1byte=l2byte=l3byte=l4byte=0;
37+
}
38+
39+
CM::~CM()
40+
{
41+
if (myMatchModel) delete myMatchModel,myMatchModel=NULL;
42+
if (myWordModel) delete myWordModel,myWordModel=NULL;
43+
if (mySparseModel) delete mySparseModel,mySparseModel=NULL;
44+
if (myRecordModel) delete myRecordModel,myRecordModel=NULL;
45+
if (myIndirectModel) delete myIndirectModel,myIndirectModel=NULL;
46+
if (mySSEModel) delete mySSEModel,mySSEModel=NULL;
47+
}
48+
49+
inline void CM::UpdByteHash()
50+
{
51+
myPPM.UpdateNextByte(l1byte);
52+
if (Param->match_model)
53+
{
54+
myMatchModel->AddByte(l1byte);
55+
myMatchModel->FindLongestContext();
56+
}
57+
if (Param->word_model) myWordModel->AddByte(l1byte);
58+
if (Param->sparse_model) mySparseModel->AddByte(l1byte);
59+
if (Param->record_model) myRecordModel->AddByte(l1byte);
60+
if (Param->indirect_model) myIndirectModel->AddByte(l1byte);
61+
}
62+
63+
int ilog2(uint32_t val)
64+
{
65+
int n=0;
66+
while (val) {val>>=1;n++;};
67+
return n;
68+
}
69+
70+
inline void CM::UpdBitPtr()
71+
{
72+
o1ctx=(l1byte<<8)+ctx;
73+
74+
int t,len,mb;
75+
t=len=mb=0;
76+
77+
if (Param->match_model)
78+
{
79+
int n=0;
80+
myMatchModel->Predict(&p1_match,n,ctx,BitPos);
81+
len=myMatchModel->GetMatchLen();
82+
mb=myMatchModel->match_bit;
83+
t=(len>0)+(len>=8)+(len>=32);
84+
}
85+
int mix1_ctx=((l2byte>>6)<<16)+(l1byte<<8)+ctx;
86+
87+
if (!Param->maximize)
88+
{
89+
int mix0_ctx=(((t<<5)+myPPM.usedorder)<<4)+(((l1byte>>5)&7)<<1)+mb;
90+
myMixer.SetCtx(mix0_ctx,1<<13);
91+
myMixer.SetCtx(mix1_ctx,0);
92+
} else {
93+
//int mix1_ctx=((l2byte>>7)<<16)+(l1byte<<8)+ctx;
94+
//myMixer.SetCtx(mix1_ctx,1<<18);
95+
//myMixer.SetCtx(h2y(l1byte+(l2byte<<8),12)&((1<<16)-1)^ctx,1<<16);
96+
myMixer.SetCtx((myPPM.usedorder<<5)+(ilog2(len)<<1)+(BitPos==0),1<<8);
97+
myMixer.SetCtx((l1byte<<8)+ctx,1<<16);
98+
myMixer.SetCtx(ctx,256);
99+
myMixer.SetCtx(l2byte,1<<16);
100+
//myMixer.SetCtx((l3byte,1<<16);
101+
//(ilog2(len)<<1)+
102+
//myMixer.SetCtx((myPPM.usedorder<<4)+((l4byte>>6)<<2)+((BitPos==0)<<1)+(l1byte==l2byte),256);
103+
//myMixer.SetCtx(l2byte,256);
104+
//myMixer.SetCtx(l3byte,256);
105+
//myMixer.SetCtx((ilog2(len)<<1)+mb,512);
106+
}
107+
108+
109+
if (Param->final_sse)
110+
{
111+
int sse0_ctx=(((((l1byte>>6)&7)<<10)+(ctx<<2)+t)<<1)+mb;
112+
int sse1_ctx=(l1byte<<8)+ctx;
113+
int sse2_ctx=(h2y(l1byte+(l2byte<<8),12)&((1<<16)-1))^ctx;
114+
int sse3_ctx=(h2y(l1byte+(l2byte<<8)+(l3byte<<16),12)&((1<<16)-1))^ctx;
115+
116+
mySSEModel->SetCtx(0,sse0_ctx);
117+
mySSEModel->SetCtx(1,sse1_ctx);
118+
mySSEModel->SetCtx(2,sse2_ctx);
119+
mySSEModel->SetCtx(3,sse3_ctx);
120+
}
121+
}
122+
123+
int CM::Predict()
124+
{
125+
UpdBitPtr();
126+
127+
myPPM.Predict(myMixer.m,myMixer.nm,ctx);
128+
if (myMatchModel) myMixer.Add(p1_match);
129+
if (myWordModel) myWordModel->Predict(myMixer.m,myMixer.nm,ctx);
130+
if (mySparseModel) mySparseModel->Predict(myMixer.m,myMixer.nm,ctx);
131+
if (myRecordModel) myRecordModel->Predict(myMixer.m,myMixer.nm,ctx);
132+
if (myIndirectModel) myIndirectModel->Predict(myMixer.m,myMixer.nm,ctx);
133+
134+
int p_out = myMixer.Predict();
135+
if (mySSEModel) p_out=mySSEModel->Predict(p_out,ctx);
136+
return clamp(p_out,1,PSCALEm);
137+
}
138+
139+
void CM::Update(int bit)
140+
{
141+
myPPM.Update(bit);
142+
if (myMatchModel) myMatchModel->Update(bit);
143+
if (myWordModel) myWordModel->Update(bit);
144+
if (mySparseModel) mySparseModel->Update(bit);
145+
if (myRecordModel) myRecordModel->Update(bit);
146+
if (myIndirectModel) myIndirectModel->Update(bit);
147+
if (mySSEModel) mySSEModel->Update(bit,Param->final_sse_limit,Param->final_sse_mix_rate);
148+
myMixer.Update(bit);
149+
}
150+
151+
void CM::Encode(int val)
152+
{
153+
ctx=1;
154+
UpdByteHash();
155+
156+
for (BitPos=7;BitPos>=0;BitPos--)
157+
{
158+
const int bit=(val>>BitPos)&1;
159+
Coder->EncodeBitOne(Predict(),bit);
160+
Update(bit);
161+
ctx+=ctx+bit;
162+
}
163+
l4byte=l3byte;
164+
l3byte=l2byte;
165+
l2byte=l1byte;
166+
l1byte=val;
167+
//hist.AddFront(val);
168+
}
169+
170+
int CM::Decode()
171+
{
172+
ctx=1;
173+
UpdByteHash();
174+
int val=0;
175+
for (BitPos=7;BitPos>=0;BitPos--)
176+
{
177+
const int bit=Coder->DecodeBitOne(Predict());
178+
Update(bit);
179+
val+=(bit<<BitPos);
180+
ctx+=ctx+bit;
181+
}
182+
l4byte=l3byte;
183+
l3byte=l2byte;
184+
l2byte=l1byte;
185+
l1byte=val;
186+
//hist.AddFront(val);
187+
return val;
188+
}
189+

‎src/codec/cm.h‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
#ifndef _CM5_H
2+
#define _CM5_H
3+
4+
5+
#include <iostream>
6+
#include "codec.h"
7+
#include "models\match.h"
8+
#include "models\word.h"
9+
#include "models\ppm.h"
10+
#include "models\fsm.h"
11+
#include "models\sparse.h"
12+
#include "models\record.h"
13+
#include "models\indirect.h"
14+
#include "models\mix.h"
15+
#include "models\sse.h"
16+
#include "..\common\histbuffer.h"
17+
#include "..\common\model\counter.h"
18+
#include "..\common\model\mixer.h"
19+
#include "..\common\model\map.h"
20+
#include "..\common\hashtable.h"
21+
22+
//enum {MAXMODEL=8};
23+
24+
class TCMParam
25+
{
26+
public:
27+
enum {MAXORDER=32};
28+
TCMParam()
29+
{
30+
memhash = 26;
31+
maxorder = 6;
32+
//counter_rate = 32;
33+
34+
match_model=final_sse=word_model=sparse_model=record_model=indirect_model=false;
35+
36+
match_win_size=18;
37+
match_model_rate=0.008*PSCALE;
38+
39+
counter_limit=100;
40+
41+
maximize=false;
42+
43+
word_model_mem=23;
44+
word_model_limit=100;
45+
46+
sparse_model_mem=22;
47+
sparse_model_limit=100;
48+
49+
record_model_mem=22;
50+
record_model_limit=100;
51+
52+
indirect_model_mem=22;
53+
indirect_model_limit=100;
54+
55+
final_mix_rate=0.001*WSCALE;
56+
final_chain_rate=0.005*WSCALE;
57+
final_sse_limit = 100;
58+
final_sse_mix_rate=0.005*WSCALE;
59+
};
60+
61+
bool match_model,word_model,sparse_model,final_sse,record_model,indirect_model;
62+
bool maximize;
63+
int maxorder;
64+
int memhash;
65+
int counter_limit;
66+
int match_win_size,match_model_rate;
67+
int sparse_model_mem,sparse_model_limit;
68+
int word_model_mem,word_model_limit;
69+
int record_model_mem,record_model_limit;
70+
int indirect_model_mem,indirect_model_limit;
71+
int final_mix_rate,final_chain_rate;
72+
int final_sse_limit,final_sse_mix_rate;
73+
};
74+
75+
class CM : public Encoder {
76+
public:
77+
CM(BaseRangeCoder *RCoder,TCMParam *CMParam,bool verbose);
78+
~CM();
79+
void Encode(int val);
80+
int Decode();
81+
int GetBytesMatched(){return myMatchModel?myMatchModel->BytesMatched:0;};
82+
protected:
83+
int Predict();
84+
void Update(int bit);
85+
void UpdByteHash();
86+
inline void UpdBitPtr();
87+
88+
int maxorder;
89+
int ctx,BitPos,o1ctx;
90+
91+
TCMParam *Param;
92+
uint8_t l1byte,l2byte,l3byte,l4byte;
93+
WordModel *myWordModel;
94+
MatchModel *myMatchModel;
95+
SparseModel *mySparseModel;
96+
RecordModel *myRecordModel;
97+
IndirectModel *myIndirectModel;
98+
SSEModel *mySSEModel;
99+
PPMModel myPPM;
100+
Mixer myMixer;
101+
int p1_match;
102+
};
103+
104+
#endif

‎src/codec/codec.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#ifndef _CODEC_H
2+
#define _CODEC_H
3+
4+
#include "..\global.h"
5+
#include "..\common\rangecoder.h"
6+
7+
class Encoder {
8+
public:
9+
Encoder(BaseRangeCoder *RCoder):Coder(RCoder){};
10+
virtual void Encode(int val)=0; // encode byte value
11+
virtual int Decode()=0;
12+
protected:
13+
BaseRangeCoder *Coder;
14+
};
15+
16+
#endif

0 commit comments

Comments
 (0)