1 /* This file is part of Metaproxy.
2 Copyright (C) 2005-2013 Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 #include <metaproxy/filter.hpp>
21 #include <metaproxy/package.hpp>
22 #include <metaproxy/util.hpp>
23 #include "filter_http_rewrite.hpp"
24 #include "html_parser.hpp"
30 #include <boost/regex.hpp>
31 #include <boost/lexical_cast.hpp>
32 #include <boost/algorithm/string.hpp>
36 namespace mp = metaproxy_1;
37 namespace yf = mp::filter;
39 namespace metaproxy_1 {
41 class HttpRewrite::Replace {
46 std::map<int, std::string> group_index;
48 const std::map<std::string, std::string> & vars) const;
49 void parse_groups(std::string pattern);
52 class HttpRewrite::Rule {
54 std::list<Replace> replace_list;
56 std::map<std::string, std::string> &vars,
57 std::string &txt, bool anchor,
58 std::list<boost::regex> &skip_list);
60 class HttpRewrite::Within {
68 bool exec(std::map<std::string, std::string> &vars,
69 std::string &txt, bool anchor,
70 std::list<boost::regex> &skip_list) const;
73 class HttpRewrite::Content {
76 boost::regex content_re;
77 std::list<Within> within_list;
78 void configure(const xmlNode *ptr,
79 std::map<std::string, RulePtr > &rules);
80 void quoted_literal(std::string &content,
81 std::map<std::string, std::string> &vars,
82 std::list<boost::regex> & skip_list) const;
83 void parse(int verbose, std::string &content,
84 std::map<std::string, std::string> & vars,
85 std::list<boost::regex> & skip_list ) const;
87 class HttpRewrite::Phase {
91 std::list<Content> content_list;
92 std::list<boost::regex> skip_list;
93 void read_skip_headers(Z_HTTP_Request *hreq);
94 void rewrite_reqline(mp::odr & o, Z_HTTP_Request *hreq,
95 std::map<std::string, std::string> & vars) const;
96 void rewrite_headers(mp::odr & o, Z_HTTP_Header *headers,
97 std::map<std::string, std::string> & vars) const;
98 void rewrite_body(mp::odr & o,
99 const char *content_type,
100 char **content_buf, int *content_len,
101 std::map<std::string, std::string> & vars,
102 std::list<boost::regex> & skip_list ) const;
104 class HttpRewrite::Event : public HTMLParserEvent {
105 void openTagStart(const char *tag, int tag_len);
106 void anyTagEnd(const char *tag, int tag_len, int close_it);
107 void attribute(const char *tag, int tag_len,
108 const char *attr, int attr_len,
109 const char *value, int val_len,
111 void closeTag(const char *tag, int tag_len);
112 void text(const char *value, int len);
113 const Content *m_content;
115 std::stack<std::list<Within>::const_iterator> s_within;
116 std::map<std::string, std::string> &m_vars;
117 std::list<boost::regex> & m_skips;
119 Event(const Content *p,
120 std::map<std::string, std::string> &vars,
121 std::list<boost::regex> & skip_list );
123 const char *result();
128 yf::HttpRewrite::HttpRewrite() :
129 req_phase(new Phase), res_phase(new Phase)
133 yf::HttpRewrite::~HttpRewrite()
137 void yf::HttpRewrite::process(mp::Package & package) const
139 yaz_log(YLOG_LOG, "HttpRewrite begins....");
140 Z_GDU *gdu = package.request().get();
141 //map of request/response vars
142 std::map<std::string, std::string> vars;
143 //we have an http req
144 if (gdu && gdu->which == Z_GDU_HTTP_Request)
146 Z_HTTP_Request *hreq = gdu->u.HTTP_Request;
148 req_phase->rewrite_reqline(o, hreq, vars);
149 res_phase->read_skip_headers(hreq);
150 yaz_log(YLOG_LOG, ">> Request headers");
151 req_phase->rewrite_headers(o, hreq->headers, vars);
152 req_phase->rewrite_body(o,
153 z_HTTP_header_lookup(hreq->headers,
155 &hreq->content_buf, &hreq->content_len,
156 vars, res_phase->skip_list);
157 // TODO skip_list does not really belong in the phase. More like
158 // HttpRewrite itself!
159 package.request() = gdu;
162 gdu = package.response().get();
163 if (gdu && gdu->which == Z_GDU_HTTP_Response)
165 Z_HTTP_Response *hres = gdu->u.HTTP_Response;
166 yaz_log(YLOG_LOG, "Response code %d", hres->code);
168 yaz_log(YLOG_LOG, "<< Respose headers");
169 res_phase->rewrite_headers(o, hres->headers, vars);
170 res_phase->rewrite_body(o,
171 z_HTTP_header_lookup(hres->headers,
173 &hres->content_buf, &hres->content_len,
174 vars, res_phase->skip_list);
175 package.response() = gdu;
179 // Read (and remove) the X-Metaproxy-SkipLink headers
180 void yf::HttpRewrite::Phase::read_skip_headers(Z_HTTP_Request *hreq)
183 std::string url(hreq->path);
184 if ( url.substr(0,7) != "http://" )
185 { // path was relative, as it usually is
186 const char *host = z_HTTP_header_lookup(hreq->headers, "Host");
188 url = "http://" + std::string(host) + hreq->path ;
191 while ( const char *hv = z_HTTP_header_remove( &(hreq->headers),
192 "X-Metaproxy-SkipLink") )
194 yaz_log(YLOG_LOG,"Found SkipLink '%s'", hv );
195 const char *p = strchr(hv,' ');
197 continue; // should not happen
198 std::string page(hv,p);
199 std::string link(p+1);
200 boost::regex pagere(page);
201 if ( boost::regex_search(url, pagere) )
203 yaz_log(YLOG_LOG,"SkipLink '%s' matches URL %s",
204 page.c_str(), url.c_str() );
205 boost::regex linkre(link);
206 skip_list.push_back(linkre);
210 yaz_log(YLOG_LOG,"SkipLink ignored, '%s' does not match '%s'",
211 url.c_str(), page.c_str() );
217 void yf::HttpRewrite::Phase::rewrite_reqline (mp::odr & o,
218 Z_HTTP_Request *hreq,
219 std::map<std::string, std::string> & vars) const
221 //rewrite the request line
223 if (strstr(hreq->path, "http://") == hreq->path)
225 yaz_log(YLOG_LOG, "Path in the method line is absolute, "
226 "possibly a proxy request");
231 //TODO what about proto
232 const char *host = z_HTTP_header_lookup(hreq->headers, "Host");
241 std::list<Content>::const_iterator cit = content_list.begin();
242 for (; cit != content_list.end(); cit++)
243 if (cit->type == "headers")
246 if (cit == content_list.end())
249 std::list<Within>::const_iterator it = cit->within_list.begin();
250 for (; it != cit->within_list.end(); it++)
253 yaz_log(YLOG_LOG, "Proxy request URL is %s", path.c_str());
254 std::list<boost::regex> dummy_skip_list; // no skips here!
255 if (it->exec(vars, path, true, dummy_skip_list))
257 yaz_log(YLOG_LOG, "Rewritten request URL is %s", path.c_str());
258 hreq->path = odr_strdup(o, path.c_str());
263 void yf::HttpRewrite::Phase::rewrite_headers(mp::odr & o,
264 Z_HTTP_Header *headers,
265 std::map<std::string, std::string> & vars ) const
267 std::list<Content>::const_iterator cit = content_list.begin();
268 for (; cit != content_list.end(); cit++)
269 if (cit->type == "headers")
272 if (cit == content_list.end())
275 for (Z_HTTP_Header *header = headers; header; header = header->next)
277 std::list<Within>::const_iterator it = cit->within_list.begin();
278 for (; it != cit->within_list.end(); it++)
280 if (!it->header.empty() &&
281 regex_match(header->name, it->header))
283 // Match and replace only the header value
284 std::string hval(header->value);
285 std::list<boost::regex> dummy_skip_list; // no skips here!
286 if (it->exec(vars, hval, true, dummy_skip_list))
288 header->value = odr_strdup(o, hval.c_str());
295 void yf::HttpRewrite::Phase::rewrite_body(
297 const char *content_type,
300 std::map<std::string, std::string> & vars,
301 std::list<boost::regex> & skip_list ) const
303 if (*content_len == 0)
306 yaz_log(YLOG_LOG, "rewrite_body: null content_type, can not rewrite");
309 std::list<Content>::const_iterator cit = content_list.begin();
310 for (; cit != content_list.end(); cit++)
312 yaz_log(YLOG_LOG, "rewrite_body: content_type=%s type=%s",
313 content_type, cit->type.c_str());
314 if (cit->type != "headers"
315 && regex_match(content_type, cit->content_re))
318 if (cit == content_list.end()) {
319 yaz_log(YLOG_LOG,"rewrite_body: No content rule matched %s, not rewriting",
325 for (i = 0; i < *content_len; i++)
326 if ((*content_buf)[i] == 0) {
327 yaz_log(YLOG_LOG,"rewrite_body: Looks like binary stuff, not rewriting");
328 return; // binary content. skip
331 std::string content(*content_buf, *content_len);
332 cit->parse(m_verbose, content, vars, skip_list);
333 *content_buf = odr_strdup(o, content.c_str());
334 *content_len = strlen(*content_buf);
337 yf::HttpRewrite::Event::Event(const Content *p,
338 std::map<std::string, std::string> & vars,
339 std::list<boost::regex> & skip_list
340 ) : m_content(p), m_vars(vars), m_skips(skip_list)
345 yf::HttpRewrite::Event::~Event()
350 const char *yf::HttpRewrite::Event::result()
352 return wrbuf_cstr(m_w);
355 void yf::HttpRewrite::Event::openTagStart(const char *tag, int tag_len)
357 wrbuf_putc(m_w, '<');
358 wrbuf_write(m_w, tag, tag_len);
360 std::string t(tag, tag_len);
361 std::list<Within>::const_iterator it = m_content->within_list.begin();
362 for (; it != m_content->within_list.end(); it++)
364 if (!it->tag.empty() && regex_match(t, it->tag))
366 if (!it->attr.empty() && regex_match("#text", it->attr))
375 void yf::HttpRewrite::Event::anyTagEnd(const char *tag, int tag_len,
380 if (!s_within.empty())
382 std::list<Within>::const_iterator it = s_within.top();
383 std::string t(tag, tag_len);
384 if (regex_match(t, it->tag))
389 wrbuf_putc(m_w, '/');
390 wrbuf_putc(m_w, '>');
393 void yf::HttpRewrite::Event::attribute(const char *tag, int tag_len,
394 const char *attr, int attr_len,
395 const char *value, int val_len,
398 std::list<Within>::const_iterator it = m_content->within_list.begin();
401 for (; it != m_content->within_list.end(); it++)
403 std::string t(tag, tag_len);
404 if (it->tag.empty() || regex_match(t, it->tag))
406 std::string a(attr, attr_len);
407 if (!it->attr.empty() && regex_match(a, it->attr))
414 wrbuf_putc(m_w, ' ');
415 wrbuf_write(m_w, attr, attr_len);
418 wrbuf_puts(m_w, "=");
419 wrbuf_puts(m_w, sep);
424 std::string s(value, val_len);
425 it->exec(m_vars, s, true, m_skips);
426 wrbuf_puts(m_w, s.c_str());
429 wrbuf_write(m_w, value, val_len);
430 wrbuf_puts(m_w, sep);
434 void yf::HttpRewrite::Event::closeTag(const char *tag, int tag_len)
436 if (!s_within.empty())
438 std::list<Within>::const_iterator it = s_within.top();
439 std::string t(tag, tag_len);
440 if (regex_match(t, it->tag))
443 wrbuf_puts(m_w, "</");
444 wrbuf_write(m_w, tag, tag_len);
447 void yf::HttpRewrite::Event::text(const char *value, int len)
449 std::list<Within>::const_iterator it = m_content->within_list.end();
450 if (!s_within.empty())
452 if (it != m_content->within_list.end())
454 std::string s(value, len);
455 it->exec(m_vars, s, false, m_skips);
456 wrbuf_puts(m_w, s.c_str());
459 wrbuf_write(m_w, value, len);
462 static bool embed_quoted_literal(
463 std::string &content,
464 std::map<std::string, std::string> &vars,
465 mp::filter::HttpRewrite::RulePtr ruleptr,
467 std::list<boost::regex> &skip_list)
469 bool replace = false;
471 const char *cp = content.c_str();
472 const char *cp0 = cp;
475 if (html_context && !strncmp(cp, """, 6))
478 res.append(cp0, cp - cp0);
482 if (!strncmp(cp, """, 6))
490 std::string s(cp0, cp - cp0);
491 if (ruleptr->test_patterns(vars, s, true, skip_list))
496 else if (*cp == '"' || *cp == '\'')
500 res.append(cp0, cp - cp0);
504 if (cp[-1] != '\\' && *cp == m)
512 std::string s(cp0, cp - cp0);
513 if (ruleptr->test_patterns(vars, s, true, skip_list))
518 else if (*cp == '/' && cp[1] == '/')
520 while (cp[1] && cp[1] != '\n')
525 res.append(cp0, cp - cp0);
530 bool yf::HttpRewrite::Within::exec(
531 std::map<std::string, std::string> & vars,
532 std::string & txt, bool anchor,
533 std::list<boost::regex> & skip_list) const
535 if (type == "quoted-literal")
537 return embed_quoted_literal(txt, vars, rule, true, skip_list);
541 return rule->test_patterns(vars, txt, anchor, skip_list);
545 bool yf::HttpRewrite::Rule::test_patterns(
546 std::map<std::string, std::string> & vars,
547 std::string & txt, bool anchor,
548 std::list<boost::regex> & skip_list )
550 bool replaces = false;
553 std::string::const_iterator start, end;
558 std::list<Replace>::iterator bit = replace_list.end();
560 bool match_one = false;
562 std::list<Replace>::iterator it = replace_list.begin();
563 for (; it != replace_list.end(); it++)
565 if (it->start_anchor && !first)
568 if (regex_search(start, end, what, it->re))
570 if (!match_one || what[0].first < bwhat[0].first)
584 for (i = 1; i < bwhat.size(); ++i)
586 //check if the group is named
587 std::map<int, std::string>::const_iterator git
588 = bit->group_index.find(i);
589 if (git != bit->group_index.end())
591 vars[git->second] = bwhat[i];
595 // Compare against skip_list
596 bool skipthis = false;
597 std::list<boost::regex>::iterator si = skip_list.begin();
598 for ( ; si != skip_list.end(); si++) {
599 if ( boost::regex_search(bwhat.str(0), *si) )
605 //prepare replacement string
606 std::string rvalue = bit->sub_vars(vars);
607 out.append(start, bwhat[0].first);
610 yaz_log(YLOG_LOG,"! Not rewriting '%s', skiplist match",
611 bwhat.str(0).c_str() );
612 out.append(bwhat.str(0).c_str());
616 yaz_log(YLOG_LOG, "! Rewritten '%s' to '%s'",
617 bwhat.str(0).c_str(), rvalue.c_str());
620 start = bwhat[0].second; //move search forward
622 out.append(start, end);
627 void yf::HttpRewrite::Replace::parse_groups(std::string pattern)
631 const std::string &str = pattern;
633 start_anchor = str[0] == '^';
634 yaz_log(YLOG_LOG, "Parsing groups from '%s'", str.c_str());
635 for (size_t i = 0; i < str.size(); ++i)
638 if (!esc && str[i] == '\\')
643 if (!esc && str[i] == '(') //group starts
646 if (i+1 < str.size() && str[i+1] == '?') //group with attrs
649 if (i+1 < str.size() && str[i+1] == ':') //non-capturing
651 if (gnum > 0) gnum--;
657 if (i+1 < str.size() && str[i+1] == 'P') //optional, python
659 if (i+1 < str.size() && str[i+1] == '<') //named
664 while (++i < str.size())
666 if (str[i] == '>') { term = true; break; }
667 if (!isalnum(str[i]))
668 throw mp::filter::FilterException
669 ("Only alphanumeric chars allowed, found "
673 + boost::lexical_cast<std::string>(i));
677 throw mp::filter::FilterException
678 ("Unterminated group name '" + gname
679 + " in '" + str +"'");
680 group_index[gnum] = gname;
681 yaz_log(YLOG_LOG, "Found named group '%s' at $%d",
682 gname.c_str(), gnum);
691 std::string yf::HttpRewrite::Replace::sub_vars(
692 const std::map<std::string, std::string> & vars) const
696 const std::string & in = recipe;
697 for (size_t i = 0; i < in.size(); ++i)
699 if (!esc && in[i] == '\\')
704 if (!esc && in[i] == '$') //var
706 if (i+1 < in.size() && in[i+1] == '{') //ref prefix
711 while (++i < in.size())
713 if (in[i] == '}') { term = true; break; }
716 if (!term) throw mp::filter::FilterException
717 ("Unterminated var ref in '"+in+"' at "
718 + boost::lexical_cast<std::string>(i));
719 std::map<std::string, std::string>::const_iterator it
721 if (it != vars.end())
728 throw mp::filter::FilterException
729 ("Malformed or trimmed var ref in '"
730 +in+"' at "+boost::lexical_cast<std::string>(i));
741 yf::HttpRewrite::Phase::Phase() : m_verbose(0)
745 void yf::HttpRewrite::Content::parse(
747 std::string &content,
748 std::map<std::string, std::string> &vars,
749 std::list<boost::regex> & skip_list ) const
754 Event ev(this, vars, skip_list);
756 parser.set_verbose(verbose);
758 parser.parse(ev, content.c_str());
759 content = ev.result();
761 if (type == "quoted-literal")
763 quoted_literal(content, vars, skip_list);
767 void yf::HttpRewrite::Content::quoted_literal(
768 std::string &content,
769 std::map<std::string, std::string> &vars,
770 std::list<boost::regex> & skip_list ) const
772 std::list<Within>::const_iterator it = within_list.begin();
773 if (it != within_list.end())
774 embed_quoted_literal(content, vars, it->rule, false, skip_list);
777 void yf::HttpRewrite::Content::configure(
778 const xmlNode *ptr, std::map<std::string, RulePtr > &rules)
780 for (; ptr; ptr = ptr->next)
782 if (ptr->type != XML_ELEMENT_NODE)
784 if (!strcmp((const char *) ptr->name, "within"))
786 static const char *names[7] =
787 { "header", "attr", "tag", "rule", "reqline", "type", 0 };
788 std::string values[6];
789 mp::xml::parse_attr(ptr, names, values);
791 if (values[0].length() > 0)
792 w.header.assign(values[0], boost::regex_constants::icase);
793 if (values[1].length() > 0)
794 w.attr.assign(values[1], boost::regex_constants::icase);
795 if (values[2].length() > 0)
796 w.tag.assign(values[2], boost::regex_constants::icase);
798 std::vector<std::string> rulenames;
799 boost::split(rulenames, values[3], boost::is_any_of(","));
800 if (rulenames.size() == 0)
802 throw mp::filter::FilterException
803 ("Empty rule in '" + values[3] +
804 "' in http_rewrite filter");
806 else if (rulenames.size() == 1)
808 std::map<std::string,RulePtr>::const_iterator it =
809 rules.find(rulenames[0]);
810 if (it == rules.end())
811 throw mp::filter::FilterException
812 ("Reference to non-existing rule '" + rulenames[0] +
813 "' in http_rewrite filter");
819 RulePtr rule(new Rule);
821 for (i = 0; i < rulenames.size(); i++)
823 std::map<std::string,RulePtr>::const_iterator it =
824 rules.find(rulenames[i]);
825 if (it == rules.end())
826 throw mp::filter::FilterException
827 ("Reference to non-existing rule '" + rulenames[i] +
828 "' in http_rewrite filter");
829 RulePtr subRule = it->second;
830 std::list<Replace>::iterator rit =
831 subRule->replace_list.begin();
832 for (; rit != subRule->replace_list.end(); rit++)
833 rule->replace_list.push_back(*rit);
837 w.reqline = values[4] == "1";
839 if (w.type.empty() || w.type == "quoted-literal")
842 throw mp::filter::FilterException
843 ("within type must be quoted-literal or none in "
844 " in http_rewrite filter");
845 within_list.push_back(w);
850 void yf::HttpRewrite::configure_phase(const xmlNode *ptr, Phase &phase)
852 static const char *names[2] = { "verbose", 0 };
853 std::string values[1];
855 mp::xml::parse_attr(ptr, names, values);
857 phase.m_verbose = atoi(values[0].c_str());
859 std::map<std::string, RulePtr > rules;
860 for (ptr = ptr->children; ptr; ptr = ptr->next)
862 if (ptr->type != XML_ELEMENT_NODE)
864 else if (!strcmp((const char *) ptr->name, "rule"))
866 static const char *names[2] = { "name", 0 };
867 std::string values[1];
868 values[0] = "default";
869 mp::xml::parse_attr(ptr, names, values);
871 RulePtr rule(new Rule);
872 for (xmlNode *p = ptr->children; p; p = p->next)
874 if (p->type != XML_ELEMENT_NODE)
876 if (!strcmp((const char *) p->name, "rewrite"))
880 const struct _xmlAttr *attr;
881 for (attr = p->properties; attr; attr = attr->next)
883 if (!strcmp((const char *) attr->name, "from"))
884 from = mp::xml::get_text(attr->children);
885 else if (!strcmp((const char *) attr->name, "to"))
886 replace.recipe = mp::xml::get_text(attr->children);
888 throw mp::filter::FilterException
890 + std::string((const char *) attr->name)
891 + " in rewrite section of http_rewrite");
893 yaz_log(YLOG_LOG, "Found rewrite rule from '%s' to '%s'",
894 from.c_str(), replace.recipe.c_str());
897 replace.parse_groups(from);
898 rule->replace_list.push_back(replace);
902 throw mp::filter::FilterException
904 + std::string((const char *) p->name)
905 + " in http_rewrite filter");
907 rules[values[0]] = rule;
909 else if (!strcmp((const char *) ptr->name, "content"))
911 static const char *names[3] =
912 { "type", "mime", 0 };
913 std::string values[2];
914 mp::xml::parse_attr(ptr, names, values);
915 if (values[0].empty())
917 throw mp::filter::FilterException
918 ("Missing attribute, type for for element "
919 + std::string((const char *) ptr->name)
920 + " in http_rewrite filter");
925 if (!values[1].empty())
926 c.content_re.assign(values[1], boost::regex::icase);
927 c.configure(ptr->children, rules);
928 phase.content_list.push_back(c);
932 throw mp::filter::FilterException
934 + std::string((const char *) ptr->name)
935 + " in http_rewrite filter");
940 void yf::HttpRewrite::configure(const xmlNode * ptr, bool test_only,
943 for (ptr = ptr->children; ptr; ptr = ptr->next)
945 if (ptr->type != XML_ELEMENT_NODE)
947 else if (!strcmp((const char *) ptr->name, "request"))
949 configure_phase(ptr, *req_phase);
951 else if (!strcmp((const char *) ptr->name, "response"))
953 configure_phase(ptr, *res_phase);
957 throw mp::filter::FilterException
959 + std::string((const char *) ptr->name)
960 + " in http_rewrite1 filter");
965 static mp::filter::Base* filter_creator()
967 return new mp::filter::HttpRewrite;
971 struct metaproxy_1_filter_struct metaproxy_1_filter_http_rewrite = {
982 * c-file-style: "Stroustrup"
983 * indent-tabs-mode: nil
985 * vim: shiftwidth=4 tabstop=8 expandtab