Ye_Ol_Pi_Shack
/
CppCMS
2
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 1 Wiki Activity
Browse Source

Testing of filter added - incomplete yet

master
Artyom Beilis 8 years ago
parent
df2ac1736b
commit
e65cee73a9
13 changed files with 597 additions and 39 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -0
      CMakeLists.txt
  2. +0
    -2
      cppcms/applications_pool.h
  3. +0
    -4
      cppcms/http_content_filter.h
  4. +49
    -0
      cppcms/http_context.h
  5. +19
    -3
      cppcms/http_request.h
  6. +2
    -0
      private/cached_settings.h
  7. +4
    -0
      src/cgi_api.cpp
  8. +1
    -14
      src/http_content_filter.cpp
  9. +33
    -5
      src/http_context.cpp
  10. +53
    -11
      src/http_request.cpp
  11. +222
    -0
      tests/filter_test.cpp
  12. +14
    -0
      tests/filter_test.js
  13. +199
    -0
      tests/filter_test.py

+ 1
- 0
CMakeLists.txt View File

@@ -706,6 +706,7 @@ set(ALL_TESTS
hash_map_test
response_test
file_buffer_test
filter_test
)

if(NOT DISABLE_PREFORK_CACHE AND NOT IS_WINDOWS)


+ 0
- 2
cppcms/applications_pool.h View File

@@ -44,8 +44,6 @@ namespace cppcms {
namespace app {
static const int synchronous = 0x0000; ///< Synchronous application
static const int asynchronous = 0x0001; ///< Asynchronous application that operates in asynchronous mode
// TBD
//static const int content_filter = 0x0002; ///< Asynchronous application that validates incoming content during upload

static const int op_mode_mask = 0x000F; /// mask to select sync vs async flags



+ 0
- 4
cppcms/http_content_filter.h View File

@@ -33,15 +33,11 @@ namespace http {
/// Abort
///
abort_upload(int status_code);
abort_upload(int status_code,std::string const &message);
virtual ~abort_upload() throw();

int code() const;
std::string message() const;

private:
int code_;
std::string message_;
};




+ 49
- 0
cppcms/http_context.h View File

@@ -190,7 +190,56 @@ namespace cppcms {
/// This function can be called from any thread
///
void submit_to_asynchronous_application(booster::intrusive_ptr<application> app,std::string const &matched_url);

private:
struct holder { virtual ~holder() {} };
template<typename T>
struct specific_holder : public holder {
specific_holder(T *ptr) : p(ptr) {}
virtual ~specific_holder() {}
booster::hold_ptr<T> p;
};
public:
template<typename T>
T *get_specific()
{
specific_holder<T> *sh=dynamic_cast<specific_holder<T> *>(get_holder());
if(!sh)
return 0;
return sh->p.get();
}
template<typename T>
void reset_specific(T *ptr = 0)
{
if(ptr == 0) {
set_holder(0);
return;
}
specific_holder<T> *sh=dynamic_cast<specific_holder<T> *>(get_holder());
if(sh) {
sh->p.reset(ptr);
}
else {
specific_holder<T> *sh = new specific_holder<T>(ptr);
set_holder(sh);
}
}
template<typename T>
T *release_specific()
{
T *r = 0;
specific_holder<T> *sh=dynamic_cast<specific_holder<T> *>(get_holder());
if(sh) {
r = sh->p.release();
}
set_holder(0);
return r;
}
private:

void set_holder(holder *p);
holder *get_holder();
friend class impl::cgi::connection;
int on_content_progress(size_t n);
int on_headers_ready();


+ 19
- 3
cppcms/http_request.h View File

@@ -299,19 +299,33 @@ namespace http {
content_limits &limits();

///
/// Get installed content filter
/// Get installed content filter, returns 0 if it is not installed, no ownership is transfered
///
basic_content_filter *content_filter();
///
/// Install content filter, setting it to 0 would remove the filter
/// Installs content filter. If another filter installed it is removed
///
void content_filter(basic_content_filter *flt);
void set_content_filter(basic_content_filter &flt);

///
/// Installs new content filter (or removes existing), the ownership of new filter is transfered to the request object
///
void reset_content_filter(basic_content_filter *flt = 0);
///
/// Release existing content filter owned by request
///
basic_content_filter *release_content_filter();
///
/// Returns true when full request content is ready
///
bool is_ready();
///
/// Set the size of the buffer for content that isn't loaded to memory directly,
/// like for example multipart/form-data, default is defined in configuration as
/// service.input_buffer_size and defaults to 65536
///
void setbuf(int size);
public:
/// \cond INTERNAL
request(impl::cgi::connection &);
@@ -321,6 +335,8 @@ namespace http {
friend class context;
friend class impl::cgi::connection;

void set_filter(basic_content_filter *ptr,bool owns);

int on_content_start();
void on_error();
int on_content_progress(size_t n);


+ 2
- 0
private/cached_settings.h View File

@@ -55,6 +55,7 @@ namespace impl {
std::string ip;
int port;
int output_buffer_size;
int input_buffer_size;
int async_output_buffer_size;
bool disable_xpowered_by;
bool disable_xpowered_by_version;
@@ -66,6 +67,7 @@ namespace impl {
ip = v.get("service.ip","127.0.0.1");
port = v.get("service.port",8080);
output_buffer_size = v.get("service.output_buffer_size",16384);
input_buffer_size = v.get("service.input_buffer_size",65536);
async_output_buffer_size = v.get("service.async_output_buffer_size",1024);
disable_xpowered_by = v.get("service.disable_xpowered_by",false);
disable_xpowered_by_version = v.get("service.disable_xpowered_by_version",false);


+ 4
- 0
src/cgi_api.cpp View File

@@ -257,6 +257,10 @@ void connection::handle_http_error(int code,http::context *context,ehandler cons
"</body>\r\n"
"</html>\r\n";
}
else {
booster::system::error_code e;
context->response().flush_async_chunk(e);
}
async_write(booster::aio::buffer(async_chunk_),true,
mfunc_to_event_handler(
&connection::handle_http_error_eof,


+ 1
- 14
src/http_content_filter.cpp View File

@@ -16,15 +16,7 @@ namespace http {

abort_upload::abort_upload(int status_code) :
cppcms_error(http::response::status_to_string(status_code)),
code_(status_code),
message_(http::response::status_to_string(status_code))
{
}

abort_upload::abort_upload(int status_code,std::string const &message):
cppcms_error(message),
code_(status_code),
message_(message)
code_(status_code)
{
}

@@ -37,11 +29,6 @@ abort_upload::~abort_upload() throw()
{
}

std::string abort_upload::message() const
{
return message_;
}

struct content_limits::_data {};

content_limits::content_limits(impl::cached_settings const &s) :


+ 33
- 5
src/http_context.cpp View File

@@ -42,6 +42,7 @@ namespace http {
booster::shared_ptr<application_specific_pool> pool;
booster::intrusive_ptr<application> app;
std::string matched;
booster::hold_ptr<context::holder> specific;
_data(context &cntx) :
locale(cntx.connection().service().locale()),
request(cntx.connection())
@@ -56,6 +57,15 @@ context::context(booster::shared_ptr<impl::cgi::connection> conn) :
skin(service().views_pool().default_skin());
}

void context::set_holder(holder *p)
{
d->specific.reset(p);
}
context::holder *context::get_holder()
{
return d->specific.get();
}

std::string context::skin()
{
return d->skin;
@@ -264,7 +274,18 @@ void context::on_request_ready(bool error)
app.swap(d->app);

if(error) {
request().on_error();
if(app) {
try {
context_guard g(*app,*this);
request().on_error();
}
catch(std::exception const &e) {
BOOSTER_ERROR("cppcms") << "exception at request::on_error" << e.what() << booster::trace(e);
}
catch(...) {
BOOSTER_ERROR("cppcms") << "Unknown exception at request::on_error";
}
}
return;
}

@@ -352,12 +373,19 @@ void context::dispatch(booster::intrusive_ptr<application> const &app,std::strin
}
if(app->get_context()) {
if(syncronous) {
app->context().complete_response();
try {
if(syncronous) {
app->context().complete_response();
}
else {
app->context().async_complete_response();
}
}
else {
app->context().async_complete_response();
catch(...) {
app->release_context();
throw;
}
app->release_context();
}
}



+ 53
- 11
src/http_request.cpp View File

@@ -131,6 +131,7 @@ struct request::_data {
std::vector<char> post_data;
content_limits limits;
basic_content_filter *filter;
bool filter_owned;
bool filter_is_raw_content_filter;
bool filter_is_multipart_filter;
bool ready;
@@ -138,31 +139,71 @@ struct request::_data {
long long read_size;
bool read_full;
bool no_on_error;
int buffer_size;
booster::hold_ptr<cppcms::impl::multipart_parser> multipart_parser;
_data(cppcms::service &srv) :
limits(srv.cached_settings()),
filter(0),
filter_owned(false),
filter_is_raw_content_filter(false),
filter_is_multipart_filter(false),
ready(false),
content_length(0),
read_size(0),
read_full(false),
no_on_error(false)
no_on_error(false),
buffer_size(srv.cached_settings().service.input_buffer_size)
{
}
~_data()
{
if(filter_owned)
delete filter;
}
};

void request::setbuf(int size)
{
if(size < 1)
size = 1;
d->buffer_size = size;
}

basic_content_filter *request::content_filter()
{
return d->filter;
}

void request::content_filter(basic_content_filter *filter)
void request::set_content_filter(basic_content_filter &flt)
{
set_filter(&flt,false);
}
void request::reset_content_filter(basic_content_filter *flt)
{
set_filter(flt,true);
}
basic_content_filter *request::release_content_filter()
{
if(d->filter_owned) {
basic_content_filter *flt = d->filter;
d->filter = 0;
d->filter_owned = false;
return flt;
}
d->filter = 0;
return 0;
}

void request::set_filter(basic_content_filter *filter,bool owned)
{
if(d->filter && d->filter != filter && d->filter_owned) {
delete d->filter;
d->filter = 0;
}
d->filter = filter;
d->filter_is_multipart_filter = dynamic_cast<multipart_filter *>(filter) != 0;
d->filter_is_raw_content_filter = dynamic_cast<raw_content_filter *>(filter) != 0;
d->filter_owned = filter ? owned : false;
d->filter_is_multipart_filter = dynamic_cast<multipart_filter *>(d->filter) != 0;
d->filter_is_raw_content_filter = dynamic_cast<raw_content_filter *>(d->filter) != 0;
}

bool request::is_ready()
@@ -180,9 +221,10 @@ std::pair<char *,size_t > request::get_buffer()
}
else {
long long reminder = d->content_length - d->read_size;
if(static_cast<long long>(d->post_data.size()) < reminder) {
d->post_data.resize(d->content_length);
}
if(reminder < d->buffer_size)
d->post_data.resize(reminder);
else
d->post_data.resize(d->buffer_size);
if(d->post_data.size() == 0) {
std::vector<char> tmp;
tmp.swap(d->post_data);
@@ -339,6 +381,8 @@ int request::on_content_progress(size_t n)

void request::on_error()
{
if(d->no_on_error)
return;
if(d->filter) {
d->filter->on_error();
}
@@ -361,10 +405,6 @@ int request::on_content_start()
d->read_full = true;
}
else {
if(d->content_length < 65535)
d->post_data.resize(d->content_length);
else
d->post_data.resize(65536);
if(content_type_.is_multipart_form_data() && !d->filter_is_raw_content_filter) {
d->multipart_parser.reset(new multipart_parser(
d->limits.uploads_path(),
@@ -406,6 +446,8 @@ bool request::prepare()
else
d->content_length = atoll(s);
content_type_ = cppcms::http::content_type(conn_->cgetenv("CONTENT_TYPE"));
if(d->content_length == 0)
d->ready = true;
return true;
}



+ 222
- 0
tests/filter_test.cpp View File

@@ -0,0 +1,222 @@
///////////////////////////////////////////////////////////////////////////////
//
// Copyright (C) 2008-2012 Artyom Beilis (Tonkikh) <artyomtnk@yahoo.com>
//
// See accompanying file COPYING.TXT file for licensing details.
//
///////////////////////////////////////////////////////////////////////////////
#include <cppcms/service.h>
#include <cppcms/application.h>
#include <cppcms/applications_pool.h>
#include <cppcms/http_request.h>
#include <cppcms/http_response.h>
#include <cppcms/http_content_filter.h>
#include <cppcms/http_context.h>
#include <cppcms/http_file.h>
#include <cppcms/url_dispatcher.h>
#include <cppcms/mount_point.h>
#include <booster/aio/deadline_timer.h>
#include <booster/posix_time.h>
#include <cppcms/json.h>
#include <booster/thread.h>
#include <iostream>
#include "client.h"
#include <sstream>
#include <booster/log.h>
#include "test.h"

int g_fail;
int total_on_error;
#define TESTNT(x) do { if(x) break; std::cerr << "FAIL: " #x " in line: " << __LINE__ << std::endl; g_fail = 1; return; } while(0)


class file_test : public cppcms::application, public cppcms::http::multipart_filter {
public:
file_test(cppcms::service &s) : cppcms::application(s)
{
}

std::string get_ref(std::string const &name)
{
int len = atoi(request().get("l_" + name).c_str());
char c = request().get("f_" + name).c_str()[0];
std::string r;
r.reserve(len);
for(int i=0;i<len;i++) {
r += c;
c++;
if(c>'z')
c='a';
}
return r;
}

struct test_data {
int on_new_file;
int on_upload_progress;
int on_data_ready;
int on_end_of_content;
int on_error;
void write(std::ostream &out)
{
out <<
"on_new_file="<<on_new_file<<"\n"
"on_upload_progress="<<on_upload_progress<<"\n"
"on_data_ready="<<on_data_ready<<"\n"
"on_end_of_content="<<on_end_of_content<<"\n"
;
}
};

test_data *data()
{
return context().get_specific<test_data>();
}

void on_new_file(cppcms::http::file &input_file)
{
data()->on_new_file++;
TESTNT(input_file.size() == 0);
std::string loc;
if((loc=request().get("save_to_"+input_file.name()))!="") {
std::cerr << "Saving to " << loc << std::endl;
input_file.output_file(loc);
}
if(request().get("abort")=="on_new_file" && input_file.name() == request().get("at"))
do_abort(502);
}
void on_upload_progress(cppcms::http::file &input_file)
{
data()->on_upload_progress++;
TESTNT(input_file.size() > 0);
std::string ref = get_ref(input_file.name());
TESTNT(size_t(input_file.size()) << ref.size());
std::ostringstream ss;
input_file.data().seekg(0);
ss<<input_file.data().rdbuf();
std::string r = ss.str();
TESTNT(r.size() == size_t(input_file.size()));
//std::cerr << "\n\nRef=" << ref << std::endl;
//std::cerr << "Res=" << r << "\n\n" << std::endl;
TESTNT(ref.substr(0,r.size())==r);
if(request().get("abort")=="on_upload_progress" && input_file.name() == request().get("at"))
do_abort(503);
}
void on_data_ready(cppcms::http::file &input_file) {
data()->on_data_ready++;
std::string ref = get_ref(input_file.name());
TESTNT(size_t(input_file.size()) == ref.size());
std::ostringstream ss;
input_file.data().seekg(0);
ss<<input_file.data().rdbuf();
std::string r = ss.str();
TESTNT(r.size() == size_t(input_file.size()));
TESTNT(ref==r);
if(request().get("abort")=="on_data_ready" && input_file.name() == request().get("at"))
do_abort(504);
}
void on_end_of_content(){
data()->on_end_of_content++;
TESTNT(request().get("fail")=="");
if(request().get("abort")=="on_end_of_content")
do_abort(505);

}
void on_error() {
data()->on_error++;
TESTNT(request().get("fail")=="1");
total_on_error++;
}
void do_abort(int code)
{
int how=atoi(request().get("how").c_str());
switch(how){
case 3:
response().setbuf(0);
case 2:
response().full_asynchronous_buffering(false);
case 1:
response().status(code);
response().set_plain_text_header();
response().out() << "at="<<request().get("abort");
case 0:
throw cppcms::http::abort_upload(code);
}
}
void main(std::string path)
{
if(path=="/total_on_error") {
response().out() << "total_on_error=" << total_on_error;
total_on_error = 0;
return;
}
if(path=="/no_content") {
TESTNT(request().is_ready());
response().out() << "no_content=1";
return;
}
if(request().get("setbuf")!="") {
request().setbuf(atoi(request().get("setbuf").c_str()));
}

TESTNT(request().is_ready() || context().get_specific<test_data>()==0);

if(!request().is_ready()) {
test_data *td = new test_data();
context().reset_specific<test_data>(td);
if(request().get("abort")=="on_headers_ready")
do_abort(501);
request().set_content_filter(*this);
}
else {
test_data *td = context().get_specific<test_data>();
TESTNT(td);
if(request().get("abort")=="") {
TESTNT(td->on_error == 0);
TESTNT(td->on_end_of_content == 1);
TESTNT(td->on_new_file == td->on_data_ready);
if(request().content_type_parsed().is_multipart_form_data())
TESTNT(td->on_new_file == int(request().post().size() + request().files().size()));
if(request().get("chunks")!="")
TESTNT(td->on_upload_progress != 0);
}
td->write(response().out());
response().out() <<
"files=" << request().files().size() << "\n"
"post=" << request().post().size() << "\n"
"total=" << request().files().size() + request().post().size() << "\n"
;

}
}
};

int main(int argc,char **argv)
{
try {
using cppcms::mount_point;
cppcms::service srv(argc,argv);

srv.applications_pool().mount( cppcms::create_pool<file_test>(),
mount_point("/upload"),
cppcms::app::asynchronous | cppcms::app::content_filter);

srv.after_fork(submitter(srv));
srv.run();
}
catch(std::exception const &e) {
std::cerr << e.what() << booster::trace(e) << std::endl;
return EXIT_FAILURE;
}
if(run_ok && !g_fail) {
std::cout << "Full Test: Ok" << std::endl;
return EXIT_SUCCESS;
}
else {
std::cout << "FAILED" << std::endl;
return EXIT_FAILURE;
}
}

+ 14
- 0
tests/filter_test.js View File

@@ -0,0 +1,14 @@

{
"service" : {
"api" : "http",
"port" : 8080,
"ip" : "127.0.0.1",
"backlog": 100,
"input_buffer_size" : 8192
},
"http" : {
"timeout" : 3,
"script_names" : [ "/upload" ]
}
}

+ 199
- 0
tests/filter_test.py View File

@@ -0,0 +1,199 @@
#!/usr/bin/env python
# coding=utf-8
#
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
#
import httplib
import sys
import re
import time
import datetime
import socket
import os

def test(x):
if not x:
raise RuntimeError("Failed")
def now():
return " " + datetime.datetime.now().strftime("%H:%M:%S.%f")

def make_content(first,length):
all=[]
for i in xrange(0,length):
all.append(first)
n=ord(first)+1
if n > ord('z'):
n=ord('a')
first = chr(n)
return ''.join(all)

def make_multipart_form_data(qs):
r=[]
fc={}
ln={}
for item in qs:
key=item.split('=')[0]
value = item.split('=')[1]
pr=key[0:2]
if pr == 'f_' or pr == 'l_':
name = key[2:]
if pr == 'f_':
fc[name]=value
else:
ln[name]=int(value)
if not name in fc or not name in ln:
continue
l=ln[name]
c=fc[name]
r.append('--123456\r\n')
r.append('Content-Type: text/plain\r\n')
r.append('Content-Disposition: form-data; name="' + name +'"\r\n\r\n')
r.append(make_content(c,l))
r.append('\r\n')
if len(r) != 0:
r.append('--123456--\r\n')
return ''.join(r)

class Conn:
num=re.compile('^[0-9]+$')
def __init__(self,path,q=[],custom_content=None):
opts={}
for item in q:
sp=item.split('=')
opts[sp[0]]=sp[1]
if len(q)==0:
self.path = path
else:
self.path = path + '?' + '&'.join(q)
if custom_content:
post=custom_content['content']
if 'content_length' in custom_content:
content_length=custom_content['content_length']
else:
content_length=len(post)
content_type=custom_content['content_type']
else:
post = make_multipart_form_data(q)
content_length = len(post)
content_type='multipart/form-data; boundary=123456'
if content_length > 0:
print 'POST ' + self.path
query = 'POST ' + self.path +' HTTP/1.0\r\n'
query += ('Content-Type: %s\r\n' % content_type)
query += ('Content-Length: %d\r\n\r\n' % content_length)
else:
print 'GET ' + self.path
query = 'GET ' + self.path + ' HTTP/1.0\r\n\r\n'
self.s=socket.socket(socket.AF_INET, socket.SOCK_STREAM);
self.s.connect(('127.0.0.1',8080))
if 'chunks' in opts:
size=int(opts['chunks'])
self.s.send(query)
pos = 0;
while pos < content_length:
if pos > 0:
time.sleep(0.05)
chunk = post[pos:pos+size]
pos+=size
self.s.send(chunk)
else:
self.s.send(query + post)

def get(self):
response = ''
while True:
tmp=self.s.recv(1000)
if len(tmp) == 0:
self.s.close()
break
response = response + tmp
r2 = response.split('\r\n\r\n')
headers=r2[0]
if len(r2) == 2:
body = r2[1]
else:
body = ''
first_header = headers.split('\r\n')[0]
if first_header.find('HTTP/1.0 ')==0:
status=int(first_header[9:12])
else:
status=0
result = {'status' : status }
result['is_html'] = body.strip().find('<')==0
if result['is_html']:
result['content']=body
else:
items=body.split('\n')
for item in items:
kv=item.split('=')
if len(kv)!=2:
continue
if Conn.num.match(kv[1]):
result[kv[0]]=int(kv[1])
else:
result[kv[0]]=kv[1]
print 'Got %s ' % result
return result

def transfer(path,q=[],custom_content=None):
c=Conn(path,q,custom_content)
return c.get()
def test_on_error_called():
test(transfer('/upload/total_on_error',[])['total_on_error']==1)

def test_upload():
r=transfer('/upload/no_content',[])
test(r['no_content']==1)
test(r['status']==200)
r=transfer('/upload',['l_1=10','f_1=a','l_2=20','f_2=t'])
test(r['files']==2)
test(r['total']==2)
test(r['on_new_file']==2)
test(r['on_upload_progress']==0)
r=transfer('/upload',['l_1=10','f_1=a','l_2=20','f_2=t','chunks=5'])
test(r['files']==2)
test(r['total']==2)
test(r['on_new_file']==2)
test(r['on_upload_progress']>2)
r=transfer('/upload',['l_1=10000','f_1=a','l_2=20000','f_2=t'])
test(r['files']==2)
test(r['total']==2)
test(r['on_new_file']==2)
test(r['on_upload_progress']>2)

code=500
for loc in ['on_headers_ready','on_new_file','on_upload_progress','on_data_ready','on_end_of_content']:
code=code+1
for how in [0,1,2,3]:
r=transfer('/upload',['l_1=50','f_1=a','abort=%s' % loc,'how=%d' % how,'setbuf=10','at=1'])
test(r['status']==code)
if how == 0:
test(r['is_html'])
else:
test(r['at']==loc)
r=transfer('/upload',[],{'content' : 'a=b&foo=bar','content_type':'application/x-www-form-urlencoded'})
test(r['files']==0)
test(r['on_new_file']==0)
test(r['on_upload_progress']==0)
test(r['on_end_of_content']==1)
test(r['post']==2)

r=transfer('/upload',['fail=1'],{'content' : 'a=b','content_length':4,'content_type':'application/x-www-form-urlencoded'})
test_on_error_called()
r=transfer('/upload',['fail=1'],{'content' : '1234567890','content_type':'multipart/form-data; boundary=123456'})
test(r['status']==400)
test_on_error_called()
r=transfer('/upload',['l_1=100','f_1=a','save_to_1=test.txt'])
test(r['status']==200)
try:
os.remove('test.txt')
except:
pass
test(open('test.txt','rb').read() == make_content('a',100))
os.remove('test.txt')

test_upload()
print "OK"

Write Preview
Loading…
Cancel
Save