Ye_Ol_Pi_Shack
/
CppCMS
2
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 1 Wiki Activity
Browse Source

Added testing of raw filter

master
Artyom Beilis 8 years ago
parent
72a96d5de0
commit
cae6ac63a4
8 changed files with 268 additions and 36 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +63
    -0
      cppcms/http_content_filter.h
  2. +3
    -3
      cppcms/http_request.h
  3. +1
    -0
      src/http_content_filter.cpp
  4. +3
    -4
      src/http_context.cpp
  5. +131
    -21
      tests/filter_test.cpp
  6. +1
    -1
      tests/filter_test.js
  7. +60
    -7
      tests/filter_test.py
  8. +6
    -0
      tests/pool_test.py

+ 63
- 0
cppcms/http_content_filter.h View File

@@ -115,6 +115,12 @@ namespace http {
booster::hold_ptr<_data> d;
};

///
/// Basic content filter that can be installed to request, all filters should be derived from this base class
///
/// Note that when `on_*` member functions of the basic_content_filter are called the original application that runs the filtering
/// has temporary installed context that can be accessed from it.
///
class CPPCMS_API basic_content_filter {
basic_content_filter(basic_content_filter const &);
void operator=(basic_content_filter const &);
@@ -122,28 +128,85 @@ namespace http {
basic_content_filter();
virtual ~basic_content_filter();

///
/// Member function that is called when entire content is read. By default does nothing.
///
/// The request can be aborted by throwing abort_upload
///
virtual void on_end_of_content();
///
/// Member function that is called in case of a error occuring during upload progress, user should not throw exception from this function but rather
/// perform cleanup procedures if needed
///
virtual void on_error();
private:
struct _data;
booster::hold_ptr<_data> d;
};

///
/// Process of any kind of generic content data.
///
/// Note: when raw_content_filter is used no content data is actually saved to request, for example request().raw_post_data() would return
/// an empty content, so it is your responsibility to store/parse whatever content you use
///
class CPPCMS_API raw_content_filter : public basic_content_filter {
public:
///
/// You must implement this member function to handle the data
///
/// A chunk of incoming data is avalible refered by data of size data_size
///
/// The request can be aborted by throwing abort_upload
///
virtual void on_data_chunk(void const *data,size_t data_size) = 0;

raw_content_filter();
virtual ~raw_content_filter();
private:
struct _raw_data;
booster::hold_ptr<_raw_data> d;
};

///
/// Filter for multipart/form-data - file upload
///
/// It allows to process/validate incomping data on the fly and make sure that for example the user is actually authorized to upload
/// such a files
///
class CPPCMS_API multipart_filter : public basic_content_filter {
public:
multipart_filter();
virtual ~multipart_filter();
///
/// New file meta-data of a form field or file is ready: the mime-type, form name and file name if provided are known, the content wasn't processed yet
///
/// Notes:
///
/// - This is the point when you can change various file properties, like location of the temporary file or specifiy output file name and more
/// - The request can be aborted by throwing abort_upload
/// - By default does nothing
///
virtual void on_new_file(http::file &input_file);
///
/// Some of the file data is available, you can access it and run some validation during upload progress.
///
/// Notes:
///
/// - This is the point when you can perform some file content validation
/// - The request can be aborted by throwing abort_upload
/// - By default does nothing
///
virtual void on_upload_progress(http::file &input_file);
///
/// The entire file data was transfered, its size wouldn't change
///
/// Notes:
///
/// - This is the point when you can save file if needed or perform final validation
/// - The request can be aborted by throwing abort_upload
/// - By default does nothing
///
virtual void on_data_ready(http::file &input_file);
private:


+ 3
- 3
cppcms/http_request.h View File

@@ -283,10 +283,10 @@ namespace http {
files_type files();
///
/// Access to raw bits of POST data. If the POST request is empty or the request_type in not POST
/// {NULL,0} will be returned.
/// Access to raw bits of POST (content) data. If the content is empty if raw_content_filter is installed
/// or multipart/form-data is handled the read_post_data().second will be 0;
///
/// Note: when processing multipart/form-data POST request this function will always return {NULL,0} as
/// Note: when processing multipart/form-data returns chunk of zero size as
/// such requests maybe huge (file uploads of multiple hundreds of MB or even GB) that are would be stored in
/// temporary files instead of memory. In order to get access to POST data you'll have to use post(), get(), or files()
/// member functions.


+ 1
- 0
src/http_content_filter.cpp View File

@@ -79,6 +79,7 @@ void multipart_filter::on_data_ready(http::file &) {}

struct raw_content_filter::_raw_data {};
void raw_content_filter::on_data_chunk(void const *,size_t) {}
raw_content_filter::raw_content_filter() {}
raw_content_filter::~raw_content_filter() {}

} // http


+ 3
- 4
src/http_context.cpp View File

@@ -22,6 +22,7 @@
#include <booster/backtrace.h>
#include <booster/aio/io_service.h>
#include <cppcms/http_content_filter.h>
#include <stdio.h>

#include "cached_settings.h"

@@ -251,13 +252,11 @@ int context::on_headers_ready()
}
}

int status = request().on_content_start();
if(status!=0)
return status;
d->pool.swap(pool);
d->matched.swap(matched);
d->app.swap(app);
return 0;
return request().on_content_start();
}

int context::on_content_progress(size_t n)


+ 131
- 21
tests/filter_test.cpp View File

@@ -30,12 +30,9 @@ int total_on_error;
#define TESTNT(x) do { if(x) break; std::cerr << "FAIL: " #x " in line: " << __LINE__ << std::endl; g_fail = 1; return; } while(0)


class file_test : public cppcms::application, public cppcms::http::multipart_filter {
class basic_test : public cppcms::application {
public:
file_test(cppcms::service &s) : cppcms::application(s)
{
}

basic_test(cppcms::service &srv) : cppcms::application(srv) {}
std::string get_ref(std::string const &name)
{
int len = atoi(request().get("l_" + name).c_str());
@@ -50,6 +47,31 @@ public:
}
return r;
}
void do_abort(int code)
{
int how=atoi(request().get("how").c_str());
switch(how){
case 3:
response().setbuf(0);
case 2:
response().full_asynchronous_buffering(false);
case 1:
response().status(code);
response().set_plain_text_header();
response().out() << "at="<<request().get("abort");
case 0:
throw cppcms::http::abort_upload(code);
}
}

};

class file_test : public basic_test, public cppcms::http::multipart_filter {
public:
file_test(cppcms::service &s) : basic_test(s)
{
}

struct test_data {
int on_new_file;
@@ -128,22 +150,6 @@ public:
total_on_error++;
}
void do_abort(int code)
{
int how=atoi(request().get("how").c_str());
switch(how){
case 3:
response().setbuf(0);
case 2:
response().full_asynchronous_buffering(false);
case 1:
response().status(code);
response().set_plain_text_header();
response().out() << "at="<<request().get("abort");
case 0:
throw cppcms::http::abort_upload(code);
}
}
void main(std::string path)
{
if(path=="/total_on_error") {
@@ -199,6 +205,106 @@ public:
};


class raw_test : public basic_test, public cppcms::http::raw_content_filter {
public:
raw_test(cppcms::service &s) : basic_test(s)
{
}

struct test_data {
int on_data_chunk;
int on_end_of_content;
int on_error;
std::string content;
test_data() : on_data_chunk(0), on_end_of_content(0), on_error(0) {}
void write(std::ostream &out)
{
out <<
"on_data_chunk="<<on_data_chunk<<"\n"
"on_end_of_content="<<on_end_of_content<<"\n"
;
}
};

test_data *data()
{
return context().get_specific<test_data>();
}

void on_data_chunk(void const *ptr,size_t data_size)
{
if(request().get("abort")=="on_data_chunk" && atoi(request().get("at").c_str()) >= int(data()->content.size()))
do_abort(502);
data()->on_data_chunk++;
data()->content.append(static_cast<char const *>(ptr),data_size);
}

void on_end_of_content(){
data()->on_end_of_content++;
TESTNT(request().get("fail")=="");
if(request().get("abort")=="on_end_of_content")
do_abort(503);

}
void on_error() {
data()->on_error++;
TESTNT(request().get("fail")=="1");
total_on_error++;
}
void main(std::string path)
{
if(path=="/total_on_error") {
response().out() << "total_on_error=" << total_on_error;
total_on_error = 0;
return;
}
if(path=="/no_content") {
TESTNT(request().is_ready());
response().out() << "no_content=1";
return;
}
if(request().get("setbuf")!="") {
request().setbuf(atoi(request().get("setbuf").c_str()));
}

TESTNT(request().is_ready() || context().get_specific<test_data>()==0);

if(!request().is_ready()) {
test_data *td = new test_data();
context().reset_specific<test_data>(td);
if(request().get("abort")=="on_headers_ready")
do_abort(501);
request().set_content_filter(*this);
std::string cl_limit,mp_limit;
if((cl_limit=request().get("cl_limit"))!="")
request().limits().content_length_limit(atoi(cl_limit.c_str()));
if((mp_limit=request().get("mp_limit"))!="")
request().limits().multipart_form_data_limit(atoi(mp_limit.c_str()));
}
else {
test_data *td = context().get_specific<test_data>();
TESTNT(td);
if(request().get("abort")=="") {
TESTNT(td->on_error == 0);
TESTNT(td->on_data_chunk >= 1);
TESTNT(td->on_end_of_content == 1);
if(request().get("chunks")!="")
TESTNT(td->on_data_chunk > 1);
if(request().get("l_1")!="")
TESTNT(td->content == get_ref("1"));
}
TESTNT(request().content_length() > 0);
TESTNT(request().raw_post_data().second == 0);
td->write(response().out());

}
}
};



int main(int argc,char **argv)
{
try {
@@ -208,6 +314,10 @@ int main(int argc,char **argv)
srv.applications_pool().mount( cppcms::create_pool<file_test>(),
mount_point("/upload"),
cppcms::app::asynchronous | cppcms::app::content_filter);
srv.applications_pool().mount( cppcms::create_pool<raw_test>(),
mount_point("/raw"),
cppcms::app::asynchronous | cppcms::app::content_filter);

srv.after_fork(submitter(srv));


+ 1
- 1
tests/filter_test.js View File

@@ -9,6 +9,6 @@
},
"http" : {
"timeout" : 3,
"script_names" : [ "/upload" ]
"script_names" : [ "/upload", "/raw" ]
}
}

+ 60
- 7
tests/filter_test.py View File

@@ -70,12 +70,17 @@ class Conn:
else:
self.path = path + '?' + '&'.join(q)
if custom_content:
post=custom_content['content']
if 'content_length' in custom_content:
content_length=custom_content['content_length']
else:
if type(custom_content) is str:
post=custom_content;
content_length=len(post)
content_type=custom_content['content_type']
content_type='text/plain'
else:
post=custom_content['content']
if 'content_length' in custom_content:
content_length=custom_content['content_length']
else:
content_length=len(post)
content_type=custom_content['content_type']
else:
post = make_multipart_form_data(q)
content_length = len(post)
@@ -143,8 +148,8 @@ def transfer(path,q=[],custom_content=None):
c=Conn(path,q,custom_content)
return c.get()
def test_on_error_called():
test(transfer('/upload/total_on_error',[])['total_on_error']==1)
def test_on_error_called(expected=1):
test(transfer('/upload/total_on_error',[])['total_on_error']==expected)

def test_upload():
r=transfer('/upload/no_content',[])
@@ -202,10 +207,58 @@ def test_upload():

test(transfer('/upload',['l_1=100','f_1=a','cl_limit=5'])['status']==200)
test(transfer('/upload',['fail=1','cl_limit=5'],{'content':'{"x":1000}','content_type':'application/json'})['status']==413)
test_on_error_called()
test(transfer('/upload',['fail=1','l_1=100','f_1=a','mp_limit=50'])['status']==413)
test_on_error_called()
test(transfer('/upload',['l_1=100','f_1=a','mp_limit=200'])['status']==200)
test(transfer('/upload',['formdata=1','l_1=100','f_1=a','l_2=200','f_2=b','mp_limit=500','cl_limit=100'])['status']==200)
test(transfer('/upload',['fail=1','formdata=1','l_1=100','f_1=a','l_2=200','f_2=b','mp_limit=500','cl_limit=99'])['status']==413)
test_on_error_called()

def test_raw():
r=transfer('/raw/no_content',[])
test(r['no_content']==1)
test(r['status']==200)
r=transfer('/raw',['l_1=10','f_1=a'],make_content('a',10))
test(r['status']==200)
r=transfer('/raw',['l_1=10','f_1=a','chunks=5'],make_content('a',10))
test(r['status']==200)
test(r['on_data_chunk']==2)
r=transfer('/raw',['l_1=10000','f_1=a'],make_content('a',10000))
test(r['status']==200)
test(r['on_data_chunk']>=2)
r=transfer('/raw',['l_1=10000','f_1=a','setbuf=10'],make_content('a',10000))
test(r['status']==200)
test(r['on_data_chunk']>=100)

code=500
for loc in ['on_headers_ready','on_data_chunk','on_end_of_content']:
code=code+1
for how in [0,1,2,3]:
for at in [0,20]:
r=transfer('/raw',['l_1=50','f_1=a','abort=%s' % loc,'how=%d' % how,'setbuf=10','at=%d' % at],make_content('a',50))
test(r['status']==code)
if how == 0:
test(r['is_html'])
else:
test(r['at']==loc)
r=transfer('/raw',['fail=1'],{'content' : 'a=b','content_length':4,'content_type':'application/x-www-form-urlencoded'})
test_on_error_called()
r=transfer('/raw',[],{'content' : 'abcdefghig','content_type':'multipart/form-data; boundary=123456'})
test(r['status']==200)
test(transfer('/raw',['l_2=100','f_2=a','cl_limit=5'])['status']==200)
test(transfer('/raw',['fail=1','cl_limit=5'],{'content':'{"x":1000}','content_type':'application/json'})['status']==413)
test_on_error_called()
test(transfer('/raw',['fail=1','l_2=100','f_2=a','mp_limit=50'])['status']==413)
test_on_error_called()
test(transfer('/raw',['l_2=100','f_2=a','mp_limit=200'])['status']==200)
test(transfer('/raw',['formdata=1','l_3=100','f_3=a','l_2=200','f_2=b','mp_limit=500','cl_limit=100'])['status']==200)
test(transfer('/raw',['formdata=1','l_3=100','f_3=a','l_2=200','f_2=b','mp_limit=500','cl_limit=99'])['status']==200)


test_upload()
test_raw()
test_on_error_called(0)
print "OK"

+ 6
- 0
tests/pool_test.py View File

@@ -93,6 +93,7 @@ def test_sync():

st=Conn('/test/unmount?id='+n).get()
Conn(n).get(exp404 = True)
time.sleep(0.1)
test(Conn('/test/stats?id='+n).get()["current"]==0)


@@ -120,6 +121,7 @@ def test_sync_prep():

Conn('/test/unmount?id='+n).get()
Conn(n).get(exp404 = True)
time.sleep(0.1)
test(Conn('/test/stats?id='+n).get()["current"]==0)


@@ -155,6 +157,7 @@ def test_sync_ts():

st=Conn('/test/unmount?id='+n).get()
Conn(n).get(exp404 = True)
time.sleep(0.1)
test(Conn('/test/stats?id='+n).get()["current"]==2)


@@ -203,6 +206,7 @@ def test_async():
st=Conn('/test/unmount?id='+n).get()
Conn(n).get(exp404 = True)
time.sleep(0.1)
test(Conn('/test/stats?id='+n).get()["current"]==0)

def test_async_prep():
@@ -224,6 +228,7 @@ def test_async_prep():

st=Conn('/test/unmount?id='+n).get()
Conn(n).get(exp404 = True)
time.sleep(0.1)
test(Conn('/test/stats?id='+n).get()["current"]==0)

def test_async_legacy():
@@ -269,6 +274,7 @@ def test_async_temporary():
st=Conn('/test/uninstall').get()
test(st["install"]==0)
time.sleep(0.1)
st=Conn('/test/stats?id=' + n).get()
test(st["total"]==1)
test(st["current"]==0)


Write Preview
Loading…
Cancel
Save