patch-realtime 789 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807880888098810881188128813881488158816881788188819882088218822882388248825882688278828882988308831883288338834883588368837883888398840884188428843884488458846884788488849885088518852885388548855885688578858885988608861886288638864886588668867886888698870887188728873887488758876887788788879888088818882888388848885888688878888888988908891889288938894889588968897889888998900890189028903890489058906890789088909891089118912891389148915891689178918891989208921892289238924892589268927892889298930893189328933893489358936893789388939894089418942894389448945894689478948894989508951895289538954895589568957895889598960896189628963896489658966896789688969897089718972897389748975897689778978897989808981898289838984898589868987898889898990899189928993899489958996899789988999900090019002900390049005900690079008900990109011901290139014901590169017901890199020902190229023902490259026902790289029903090319032903390349035903690379038903990409041904290439044904590469047904890499050905190529053905490559056905790589059906090619062906390649065906690679068906990709071907290739074907590769077907890799080908190829083908490859086908790889089909090919092909390949095909690979098909991009101910291039104910591069107910891099110911191129113911491159116911791189119912091219122912391249125912691279128912991309131913291339134913591369137913891399140914191429143914491459146914791489149915091519152915391549155915691579158915991609161916291639164916591669167916891699170917191729173917491759176917791789179918091819182918391849185918691879188918991909191919291939194919591969197919891999200920192029203920492059206920792089209921092119212921392149215921692179218921992209221922292239224922592269227922892299230923192329233923492359236923792389239924092419242924392449245924692479248924992509251925292539254925592569257925892599260926192629263926492659266926792689269927092719272927392749275927692779278927992809281928292839284928592869287928892899290929192929293929492959296929792989299930093019302930393049305930693079308930993109311931293139314931593169317931893199320932193229323932493259326932793289329933093319332933393349335933693379338933993409341934293439344934593469347934893499350935193529353935493559356935793589359936093619362936393649365936693679368936993709371937293739374937593769377937893799380938193829383938493859386938793889389939093919392939393949395939693979398939994009401940294039404940594069407940894099410941194129413941494159416941794189419942094219422942394249425942694279428942994309431943294339434943594369437943894399440944194429443944494459446944794489449945094519452945394549455945694579458945994609461946294639464946594669467946894699470947194729473947494759476947794789479948094819482948394849485948694879488948994909491949294939494949594969497949894999500950195029503950495059506950795089509951095119512951395149515951695179518951995209521952295239524952595269527952895299530953195329533953495359536953795389539954095419542954395449545954695479548954995509551955295539554955595569557955895599560956195629563956495659566956795689569957095719572957395749575957695779578957995809581958295839584958595869587958895899590959195929593959495959596959795989599960096019602960396049605960696079608960996109611961296139614961596169617961896199620962196229623962496259626962796289629963096319632963396349635963696379638963996409641964296439644964596469647964896499650965196529653965496559656965796589659966096619662966396649665966696679668966996709671967296739674967596769677967896799680968196829683968496859686968796889689969096919692969396949695969696979698969997009701970297039704970597069707970897099710971197129713971497159716971797189719972097219722972397249725972697279728972997309731973297339734973597369737973897399740974197429743974497459746974797489749975097519752975397549755975697579758975997609761976297639764976597669767976897699770977197729773977497759776977797789779978097819782978397849785978697879788978997909791979297939794979597969797979897999800980198029803980498059806980798089809981098119812981398149815981698179818981998209821982298239824982598269827982898299830983198329833983498359836983798389839984098419842984398449845984698479848984998509851985298539854985598569857985898599860986198629863986498659866986798689869987098719872987398749875987698779878987998809881988298839884988598869887988898899890989198929893989498959896989798989899990099019902990399049905990699079908990999109911991299139914991599169917991899199920992199229923992499259926992799289929993099319932993399349935993699379938993999409941994299439944994599469947994899499950995199529953995499559956995799589959996099619962996399649965996699679968996999709971997299739974997599769977997899799980998199829983998499859986998799889989999099919992999399949995999699979998999910000100011000210003100041000510006100071000810009100101001110012100131001410015100161001710018100191002010021100221002310024100251002610027100281002910030100311003210033100341003510036100371003810039100401004110042100431004410045100461004710048100491005010051100521005310054100551005610057100581005910060100611006210063100641006510066100671006810069100701007110072100731007410075100761007710078100791008010081100821008310084100851008610087100881008910090100911009210093100941009510096100971009810099101001010110102101031010410105101061010710108101091011010111101121011310114101151011610117101181011910120101211012210123101241012510126101271012810129101301013110132101331013410135101361013710138101391014010141101421014310144101451014610147101481014910150101511015210153101541015510156101571015810159101601016110162101631016410165101661016710168101691017010171101721017310174101751017610177101781017910180101811018210183101841018510186101871018810189101901019110192101931019410195101961019710198101991020010201102021020310204102051020610207102081020910210102111021210213102141021510216102171021810219102201022110222102231022410225102261022710228102291023010231102321023310234102351023610237102381023910240102411024210243102441024510246102471024810249102501025110252102531025410255102561025710258102591026010261102621026310264102651026610267102681026910270102711027210273102741027510276102771027810279102801028110282102831028410285102861028710288102891029010291102921029310294102951029610297102981029910300103011030210303103041030510306103071030810309103101031110312103131031410315103161031710318103191032010321103221032310324103251032610327103281032910330103311033210333103341033510336103371033810339103401034110342103431034410345103461034710348103491035010351103521035310354103551035610357103581035910360103611036210363103641036510366103671036810369103701037110372103731037410375103761037710378103791038010381103821038310384103851038610387103881038910390103911039210393103941039510396103971039810399104001040110402104031040410405104061040710408104091041010411104121041310414104151041610417104181041910420104211042210423104241042510426104271042810429104301043110432104331043410435104361043710438104391044010441104421044310444104451044610447104481044910450104511045210453104541045510456104571045810459104601046110462104631046410465104661046710468104691047010471104721047310474104751047610477104781047910480104811048210483104841048510486104871048810489104901049110492104931049410495104961049710498104991050010501105021050310504105051050610507105081050910510105111051210513105141051510516105171051810519105201052110522105231052410525105261052710528105291053010531105321053310534105351053610537105381053910540105411054210543105441054510546105471054810549105501055110552105531055410555105561055710558105591056010561105621056310564105651056610567105681056910570105711057210573105741057510576105771057810579105801058110582105831058410585105861058710588105891059010591105921059310594105951059610597105981059910600106011060210603106041060510606106071060810609106101061110612106131061410615106161061710618106191062010621106221062310624106251062610627106281062910630106311063210633106341063510636106371063810639106401064110642106431064410645106461064710648106491065010651106521065310654106551065610657106581065910660106611066210663106641066510666106671066810669106701067110672106731067410675106761067710678106791068010681106821068310684106851068610687106881068910690106911069210693106941069510696106971069810699107001070110702107031070410705107061070710708107091071010711107121071310714107151071610717107181071910720107211072210723107241072510726107271072810729107301073110732107331073410735107361073710738107391074010741107421074310744107451074610747107481074910750107511075210753107541075510756107571075810759107601076110762107631076410765107661076710768107691077010771107721077310774107751077610777107781077910780107811078210783107841078510786107871078810789107901079110792107931079410795107961079710798107991080010801108021080310804108051080610807108081080910810108111081210813108141081510816108171081810819108201082110822108231082410825108261082710828108291083010831108321083310834108351083610837108381083910840108411084210843108441084510846108471084810849108501085110852108531085410855108561085710858108591086010861108621086310864108651086610867108681086910870108711087210873108741087510876108771087810879108801088110882108831088410885108861088710888108891089010891108921089310894108951089610897108981089910900109011090210903109041090510906109071090810909109101091110912109131091410915109161091710918109191092010921109221092310924109251092610927109281092910930109311093210933109341093510936109371093810939109401094110942109431094410945109461094710948109491095010951109521095310954109551095610957109581095910960109611096210963109641096510966109671096810969109701097110972109731097410975109761097710978109791098010981109821098310984109851098610987109881098910990109911099210993109941099510996109971099810999110001100111002110031100411005110061100711008110091101011011110121101311014110151101611017110181101911020110211102211023110241102511026110271102811029110301103111032110331103411035110361103711038110391104011041110421104311044110451104611047110481104911050110511105211053110541105511056110571105811059110601106111062110631106411065110661106711068110691107011071110721107311074110751107611077110781107911080110811108211083110841108511086110871108811089110901109111092110931109411095110961109711098110991110011101111021110311104111051110611107111081110911110111111111211113111141111511116111171111811119111201112111122111231112411125111261112711128111291113011131111321113311134111351113611137111381113911140111411114211143111441114511146111471114811149111501115111152111531115411155111561115711158111591116011161111621116311164111651116611167111681116911170111711117211173111741117511176111771117811179111801118111182111831118411185111861118711188111891119011191111921119311194111951119611197111981119911200112011120211203112041120511206112071120811209112101121111212112131121411215112161121711218112191122011221112221122311224112251122611227112281122911230112311123211233112341123511236112371123811239112401124111242112431124411245112461124711248112491125011251112521125311254112551125611257112581125911260112611126211263112641126511266112671126811269112701127111272112731127411275112761127711278112791128011281112821128311284112851128611287112881128911290112911129211293112941129511296112971129811299113001130111302113031130411305113061130711308113091131011311113121131311314113151131611317113181131911320113211132211323113241132511326113271132811329113301133111332113331133411335113361133711338113391134011341113421134311344113451134611347113481134911350113511135211353113541135511356113571135811359113601136111362113631136411365113661136711368113691137011371113721137311374113751137611377113781137911380113811138211383113841138511386113871138811389113901139111392113931139411395113961139711398113991140011401114021140311404114051140611407114081140911410114111141211413114141141511416114171141811419114201142111422114231142411425114261142711428114291143011431114321143311434114351143611437114381143911440114411144211443114441144511446114471144811449114501145111452114531145411455114561145711458114591146011461114621146311464114651146611467114681146911470114711147211473114741147511476114771147811479114801148111482114831148411485114861148711488114891149011491114921149311494114951149611497114981149911500115011150211503115041150511506115071150811509115101151111512115131151411515115161151711518115191152011521115221152311524115251152611527115281152911530115311153211533115341153511536115371153811539115401154111542115431154411545115461154711548115491155011551115521155311554115551155611557115581155911560115611156211563115641156511566115671156811569115701157111572115731157411575115761157711578115791158011581115821158311584115851158611587115881158911590115911159211593115941159511596115971159811599116001160111602116031160411605116061160711608116091161011611116121161311614116151161611617116181161911620116211162211623116241162511626116271162811629116301163111632116331163411635116361163711638116391164011641116421164311644116451164611647116481164911650116511165211653116541165511656116571165811659116601166111662116631166411665116661166711668116691167011671116721167311674116751167611677116781167911680116811168211683116841168511686116871168811689116901169111692116931169411695116961169711698116991170011701117021170311704117051170611707117081170911710117111171211713117141171511716117171171811719117201172111722117231172411725117261172711728117291173011731117321173311734117351173611737117381173911740117411174211743117441174511746117471174811749117501175111752117531175411755117561175711758117591176011761117621176311764117651176611767117681176911770117711177211773117741177511776117771177811779117801178111782117831178411785117861178711788117891179011791117921179311794117951179611797117981179911800118011180211803118041180511806118071180811809118101181111812118131181411815118161181711818118191182011821118221182311824118251182611827118281182911830118311183211833118341183511836118371183811839118401184111842118431184411845118461184711848118491185011851118521185311854118551185611857118581185911860118611186211863118641186511866118671186811869118701187111872118731187411875118761187711878118791188011881118821188311884118851188611887118881188911890118911189211893118941189511896118971189811899119001190111902119031190411905119061190711908119091191011911119121191311914119151191611917119181191911920119211192211923119241192511926119271192811929119301193111932119331193411935119361193711938119391194011941119421194311944119451194611947119481194911950119511195211953119541195511956119571195811959119601196111962119631196411965119661196711968119691197011971119721197311974119751197611977119781197911980119811198211983119841198511986119871198811989119901199111992119931199411995119961199711998119991200012001120021200312004120051200612007120081200912010120111201212013120141201512016120171201812019120201202112022120231202412025120261202712028120291203012031120321203312034120351203612037120381203912040120411204212043120441204512046120471204812049120501205112052120531205412055120561205712058120591206012061120621206312064120651206612067120681206912070120711207212073120741207512076120771207812079120801208112082120831208412085120861208712088120891209012091120921209312094120951209612097120981209912100121011210212103121041210512106121071210812109121101211112112121131211412115121161211712118121191212012121121221212312124121251212612127121281212912130121311213212133121341213512136121371213812139121401214112142121431214412145121461214712148121491215012151121521215312154121551215612157121581215912160121611216212163121641216512166121671216812169121701217112172121731217412175121761217712178121791218012181121821218312184121851218612187121881218912190121911219212193121941219512196121971219812199122001220112202122031220412205122061220712208122091221012211122121221312214122151221612217122181221912220122211222212223122241222512226122271222812229122301223112232122331223412235122361223712238122391224012241122421224312244122451224612247122481224912250122511225212253122541225512256122571225812259122601226112262122631226412265122661226712268122691227012271122721227312274122751227612277122781227912280122811228212283122841228512286122871228812289122901229112292122931229412295122961229712298122991230012301123021230312304123051230612307123081230912310123111231212313123141231512316123171231812319123201232112322123231232412325123261232712328123291233012331123321233312334123351233612337123381233912340123411234212343123441234512346123471234812349123501235112352123531235412355123561235712358123591236012361123621236312364123651236612367123681236912370123711237212373123741237512376123771237812379123801238112382123831238412385123861238712388123891239012391123921239312394123951239612397123981239912400124011240212403124041240512406124071240812409124101241112412124131241412415124161241712418124191242012421124221242312424124251242612427124281242912430124311243212433124341243512436124371243812439124401244112442124431244412445124461244712448124491245012451124521245312454124551245612457124581245912460124611246212463124641246512466124671246812469124701247112472124731247412475124761247712478124791248012481124821248312484124851248612487124881248912490124911249212493124941249512496124971249812499125001250112502125031250412505125061250712508125091251012511125121251312514125151251612517125181251912520125211252212523125241252512526125271252812529125301253112532125331253412535125361253712538125391254012541125421254312544125451254612547125481254912550125511255212553125541255512556125571255812559125601256112562125631256412565125661256712568125691257012571125721257312574125751257612577125781257912580125811258212583125841258512586125871258812589125901259112592125931259412595125961259712598125991260012601126021260312604126051260612607126081260912610126111261212613126141261512616126171261812619126201262112622126231262412625126261262712628126291263012631126321263312634126351263612637126381263912640126411264212643126441264512646126471264812649126501265112652126531265412655126561265712658126591266012661126621266312664126651266612667126681266912670126711267212673126741267512676126771267812679126801268112682126831268412685126861268712688126891269012691126921269312694126951269612697126981269912700127011270212703127041270512706127071270812709127101271112712127131271412715127161271712718127191272012721127221272312724127251272612727127281272912730127311273212733127341273512736127371273812739127401274112742127431274412745127461274712748127491275012751127521275312754127551275612757127581275912760127611276212763127641276512766127671276812769127701277112772127731277412775127761277712778127791278012781127821278312784127851278612787127881278912790127911279212793127941279512796127971279812799128001280112802128031280412805128061280712808128091281012811128121281312814128151281612817128181281912820128211282212823128241282512826128271282812829128301283112832128331283412835128361283712838128391284012841128421284312844128451284612847128481284912850128511285212853128541285512856128571285812859128601286112862128631286412865128661286712868128691287012871128721287312874128751287612877128781287912880128811288212883128841288512886128871288812889128901289112892128931289412895128961289712898128991290012901129021290312904129051290612907129081290912910129111291212913129141291512916129171291812919129201292112922129231292412925129261292712928129291293012931129321293312934129351293612937129381293912940129411294212943129441294512946129471294812949129501295112952129531295412955129561295712958129591296012961129621296312964129651296612967129681296912970129711297212973129741297512976129771297812979129801298112982129831298412985129861298712988129891299012991129921299312994129951299612997129981299913000130011300213003130041300513006130071300813009130101301113012130131301413015130161301713018130191302013021130221302313024130251302613027130281302913030130311303213033130341303513036130371303813039130401304113042130431304413045130461304713048130491305013051130521305313054130551305613057130581305913060130611306213063130641306513066130671306813069130701307113072130731307413075130761307713078130791308013081130821308313084130851308613087130881308913090130911309213093130941309513096130971309813099131001310113102131031310413105131061310713108131091311013111131121311313114131151311613117131181311913120131211312213123131241312513126131271312813129131301313113132131331313413135131361313713138131391314013141131421314313144131451314613147131481314913150131511315213153131541315513156131571315813159131601316113162131631316413165131661316713168131691317013171131721317313174131751317613177131781317913180131811318213183131841318513186131871318813189131901319113192131931319413195131961319713198131991320013201132021320313204132051320613207132081320913210132111321213213132141321513216132171321813219132201322113222132231322413225132261322713228132291323013231132321323313234132351323613237132381323913240132411324213243132441324513246132471324813249132501325113252132531325413255132561325713258132591326013261132621326313264132651326613267132681326913270132711327213273132741327513276132771327813279132801328113282132831328413285132861328713288132891329013291132921329313294132951329613297132981329913300133011330213303133041330513306133071330813309133101331113312133131331413315133161331713318133191332013321133221332313324133251332613327133281332913330133311333213333133341333513336133371333813339133401334113342133431334413345133461334713348133491335013351133521335313354133551335613357133581335913360133611336213363133641336513366133671336813369133701337113372133731337413375133761337713378133791338013381133821338313384133851338613387133881338913390133911339213393133941339513396133971339813399134001340113402134031340413405134061340713408134091341013411134121341313414134151341613417134181341913420134211342213423134241342513426134271342813429134301343113432134331343413435134361343713438134391344013441134421344313444134451344613447134481344913450134511345213453134541345513456134571345813459134601346113462134631346413465134661346713468134691347013471134721347313474134751347613477134781347913480134811348213483134841348513486134871348813489134901349113492134931349413495134961349713498134991350013501135021350313504135051350613507135081350913510135111351213513135141351513516135171351813519135201352113522135231352413525135261352713528135291353013531135321353313534135351353613537135381353913540135411354213543135441354513546135471354813549135501355113552135531355413555135561355713558135591356013561135621356313564135651356613567135681356913570135711357213573135741357513576135771357813579135801358113582135831358413585135861358713588135891359013591135921359313594135951359613597135981359913600136011360213603136041360513606136071360813609136101361113612136131361413615136161361713618136191362013621136221362313624136251362613627136281362913630136311363213633136341363513636136371363813639136401364113642136431364413645136461364713648136491365013651136521365313654136551365613657136581365913660136611366213663136641366513666136671366813669136701367113672136731367413675136761367713678136791368013681136821368313684136851368613687136881368913690136911369213693136941369513696136971369813699137001370113702137031370413705137061370713708137091371013711137121371313714137151371613717137181371913720137211372213723137241372513726137271372813729137301373113732137331373413735137361373713738137391374013741137421374313744137451374613747137481374913750137511375213753137541375513756137571375813759137601376113762137631376413765137661376713768137691377013771137721377313774137751377613777137781377913780137811378213783137841378513786137871378813789137901379113792137931379413795137961379713798137991380013801138021380313804138051380613807138081380913810138111381213813138141381513816138171381813819138201382113822138231382413825138261382713828138291383013831138321383313834138351383613837138381383913840138411384213843138441384513846138471384813849138501385113852138531385413855138561385713858138591386013861138621386313864138651386613867138681386913870138711387213873138741387513876138771387813879138801388113882138831388413885138861388713888138891389013891138921389313894138951389613897138981389913900139011390213903139041390513906139071390813909139101391113912139131391413915139161391713918139191392013921139221392313924139251392613927139281392913930139311393213933139341393513936139371393813939139401394113942139431394413945139461394713948139491395013951139521395313954139551395613957139581395913960139611396213963139641396513966139671396813969139701397113972139731397413975139761397713978139791398013981139821398313984139851398613987139881398913990139911399213993139941399513996139971399813999140001400114002140031400414005140061400714008140091401014011140121401314014140151401614017140181401914020140211402214023140241402514026140271402814029140301403114032140331403414035140361403714038140391404014041140421404314044140451404614047140481404914050140511405214053140541405514056140571405814059140601406114062140631406414065140661406714068140691407014071140721407314074140751407614077140781407914080140811408214083140841408514086140871408814089140901409114092140931409414095140961409714098140991410014101141021410314104141051410614107141081410914110141111411214113141141411514116141171411814119141201412114122141231412414125141261412714128141291413014131141321413314134141351413614137141381413914140141411414214143141441414514146141471414814149141501415114152141531415414155141561415714158141591416014161141621416314164141651416614167141681416914170141711417214173141741417514176141771417814179141801418114182141831418414185141861418714188141891419014191141921419314194141951419614197141981419914200142011420214203142041420514206142071420814209142101421114212142131421414215142161421714218142191422014221142221422314224142251422614227142281422914230142311423214233142341423514236142371423814239142401424114242142431424414245142461424714248142491425014251142521425314254142551425614257142581425914260142611426214263142641426514266142671426814269142701427114272142731427414275142761427714278142791428014281142821428314284142851428614287142881428914290142911429214293142941429514296142971429814299143001430114302143031430414305143061430714308143091431014311143121431314314143151431614317143181431914320143211432214323143241432514326143271432814329143301433114332143331433414335143361433714338143391434014341143421434314344143451434614347143481434914350143511435214353143541435514356143571435814359143601436114362143631436414365143661436714368143691437014371143721437314374143751437614377143781437914380143811438214383143841438514386143871438814389143901439114392143931439414395143961439714398143991440014401144021440314404144051440614407144081440914410144111441214413144141441514416144171441814419144201442114422144231442414425144261442714428144291443014431144321443314434144351443614437144381443914440144411444214443144441444514446144471444814449144501445114452144531445414455144561445714458144591446014461144621446314464144651446614467144681446914470144711447214473144741447514476144771447814479144801448114482144831448414485144861448714488144891449014491144921449314494144951449614497144981449914500145011450214503145041450514506145071450814509145101451114512145131451414515145161451714518145191452014521145221452314524145251452614527145281452914530145311453214533145341453514536145371453814539145401454114542145431454414545145461454714548145491455014551145521455314554145551455614557145581455914560145611456214563145641456514566145671456814569145701457114572145731457414575145761457714578145791458014581145821458314584145851458614587145881458914590145911459214593145941459514596145971459814599146001460114602146031460414605146061460714608146091461014611146121461314614146151461614617146181461914620146211462214623146241462514626146271462814629146301463114632146331463414635146361463714638146391464014641146421464314644146451464614647146481464914650146511465214653146541465514656146571465814659146601466114662146631466414665146661466714668146691467014671146721467314674146751467614677146781467914680146811468214683146841468514686146871468814689146901469114692146931469414695146961469714698146991470014701147021470314704147051470614707147081470914710147111471214713147141471514716147171471814719147201472114722147231472414725147261472714728147291473014731147321473314734147351473614737147381473914740147411474214743147441474514746147471474814749147501475114752147531475414755147561475714758147591476014761147621476314764147651476614767147681476914770147711477214773147741477514776147771477814779147801478114782147831478414785147861478714788147891479014791147921479314794147951479614797147981479914800148011480214803148041480514806148071480814809148101481114812148131481414815148161481714818148191482014821148221482314824148251482614827148281482914830148311483214833148341483514836148371483814839148401484114842148431484414845148461484714848148491485014851148521485314854148551485614857148581485914860148611486214863148641486514866148671486814869148701487114872148731487414875148761487714878148791488014881148821488314884148851488614887148881488914890148911489214893148941489514896148971489814899149001490114902149031490414905149061490714908149091491014911149121491314914149151491614917149181491914920149211492214923149241492514926149271492814929149301493114932149331493414935149361493714938149391494014941149421494314944149451494614947149481494914950149511495214953149541495514956149571495814959149601496114962149631496414965149661496714968149691497014971149721497314974149751497614977149781497914980149811498214983149841498514986149871498814989149901499114992149931499414995149961499714998149991500015001150021500315004150051500615007150081500915010150111501215013150141501515016150171501815019150201502115022150231502415025150261502715028150291503015031150321503315034150351503615037150381503915040150411504215043150441504515046150471504815049150501505115052150531505415055150561505715058150591506015061150621506315064150651506615067150681506915070150711507215073150741507515076150771507815079150801508115082150831508415085150861508715088150891509015091150921509315094150951509615097150981509915100151011510215103151041510515106151071510815109151101511115112151131511415115151161511715118151191512015121151221512315124151251512615127151281512915130151311513215133151341513515136151371513815139151401514115142151431514415145151461514715148151491515015151151521515315154151551515615157151581515915160151611516215163151641516515166151671516815169151701517115172151731517415175151761517715178151791518015181151821518315184151851518615187151881518915190151911519215193151941519515196151971519815199152001520115202152031520415205152061520715208152091521015211152121521315214152151521615217152181521915220152211522215223152241522515226152271522815229152301523115232152331523415235152361523715238152391524015241152421524315244152451524615247152481524915250152511525215253152541525515256152571525815259152601526115262152631526415265152661526715268152691527015271152721527315274152751527615277152781527915280152811528215283152841528515286152871528815289152901529115292152931529415295152961529715298152991530015301153021530315304153051530615307153081530915310153111531215313153141531515316153171531815319153201532115322153231532415325153261532715328153291533015331153321533315334153351533615337153381533915340153411534215343153441534515346153471534815349153501535115352153531535415355153561535715358153591536015361153621536315364153651536615367153681536915370153711537215373153741537515376153771537815379153801538115382153831538415385153861538715388153891539015391153921539315394153951539615397153981539915400154011540215403154041540515406154071540815409154101541115412154131541415415154161541715418154191542015421154221542315424154251542615427154281542915430154311543215433154341543515436154371543815439154401544115442154431544415445154461544715448154491545015451154521545315454154551545615457154581545915460154611546215463154641546515466154671546815469154701547115472154731547415475154761547715478154791548015481154821548315484154851548615487154881548915490154911549215493154941549515496154971549815499155001550115502155031550415505155061550715508155091551015511155121551315514155151551615517155181551915520155211552215523155241552515526155271552815529155301553115532155331553415535155361553715538155391554015541155421554315544155451554615547155481554915550155511555215553155541555515556155571555815559155601556115562155631556415565155661556715568155691557015571155721557315574155751557615577155781557915580155811558215583155841558515586155871558815589155901559115592155931559415595155961559715598155991560015601156021560315604156051560615607156081560915610156111561215613156141561515616156171561815619156201562115622156231562415625156261562715628156291563015631156321563315634156351563615637156381563915640156411564215643156441564515646156471564815649156501565115652156531565415655156561565715658156591566015661156621566315664156651566615667156681566915670156711567215673156741567515676156771567815679156801568115682156831568415685156861568715688156891569015691156921569315694156951569615697156981569915700157011570215703157041570515706157071570815709157101571115712157131571415715157161571715718157191572015721157221572315724157251572615727157281572915730157311573215733157341573515736157371573815739157401574115742157431574415745157461574715748157491575015751157521575315754157551575615757157581575915760157611576215763157641576515766157671576815769157701577115772157731577415775157761577715778157791578015781157821578315784157851578615787157881578915790157911579215793157941579515796157971579815799158001580115802158031580415805158061580715808158091581015811158121581315814158151581615817158181581915820158211582215823158241582515826158271582815829158301583115832158331583415835158361583715838158391584015841158421584315844158451584615847158481584915850158511585215853158541585515856158571585815859158601586115862158631586415865158661586715868158691587015871158721587315874158751587615877158781587915880158811588215883158841588515886158871588815889158901589115892158931589415895158961589715898158991590015901159021590315904159051590615907159081590915910159111591215913159141591515916159171591815919159201592115922159231592415925159261592715928159291593015931159321593315934159351593615937159381593915940159411594215943159441594515946159471594815949159501595115952159531595415955159561595715958159591596015961159621596315964159651596615967159681596915970159711597215973159741597515976159771597815979159801598115982159831598415985159861598715988159891599015991159921599315994159951599615997159981599916000160011600216003160041600516006160071600816009160101601116012160131601416015160161601716018160191602016021160221602316024160251602616027160281602916030160311603216033160341603516036160371603816039160401604116042160431604416045160461604716048160491605016051160521605316054160551605616057160581605916060160611606216063160641606516066160671606816069160701607116072160731607416075160761607716078160791608016081160821608316084160851608616087160881608916090160911609216093160941609516096160971609816099161001610116102161031610416105161061610716108161091611016111161121611316114161151611616117161181611916120161211612216123161241612516126161271612816129161301613116132161331613416135161361613716138161391614016141161421614316144161451614616147161481614916150161511615216153161541615516156161571615816159161601616116162161631616416165161661616716168161691617016171161721617316174161751617616177161781617916180161811618216183161841618516186161871618816189161901619116192161931619416195161961619716198161991620016201162021620316204162051620616207162081620916210162111621216213162141621516216162171621816219162201622116222162231622416225162261622716228162291623016231162321623316234162351623616237162381623916240162411624216243162441624516246162471624816249162501625116252162531625416255162561625716258162591626016261162621626316264162651626616267162681626916270162711627216273162741627516276162771627816279162801628116282162831628416285162861628716288162891629016291162921629316294162951629616297162981629916300163011630216303163041630516306163071630816309163101631116312163131631416315163161631716318163191632016321163221632316324163251632616327163281632916330163311633216333163341633516336163371633816339163401634116342163431634416345163461634716348163491635016351163521635316354163551635616357163581635916360163611636216363163641636516366163671636816369163701637116372163731637416375163761637716378163791638016381163821638316384163851638616387163881638916390163911639216393163941639516396163971639816399164001640116402164031640416405164061640716408164091641016411164121641316414164151641616417164181641916420164211642216423164241642516426164271642816429164301643116432164331643416435164361643716438164391644016441164421644316444164451644616447164481644916450164511645216453164541645516456164571645816459164601646116462164631646416465164661646716468164691647016471164721647316474164751647616477164781647916480164811648216483164841648516486164871648816489164901649116492164931649416495164961649716498164991650016501165021650316504165051650616507165081650916510165111651216513165141651516516165171651816519165201652116522165231652416525165261652716528165291653016531165321653316534165351653616537165381653916540165411654216543165441654516546165471654816549165501655116552165531655416555165561655716558165591656016561165621656316564165651656616567165681656916570165711657216573165741657516576165771657816579165801658116582165831658416585165861658716588165891659016591165921659316594165951659616597165981659916600166011660216603166041660516606166071660816609166101661116612166131661416615166161661716618166191662016621166221662316624166251662616627166281662916630166311663216633166341663516636166371663816639166401664116642166431664416645166461664716648166491665016651166521665316654166551665616657166581665916660166611666216663166641666516666166671666816669166701667116672166731667416675166761667716678166791668016681166821668316684166851668616687166881668916690166911669216693166941669516696166971669816699167001670116702167031670416705167061670716708167091671016711167121671316714167151671616717167181671916720167211672216723167241672516726167271672816729167301673116732167331673416735167361673716738167391674016741167421674316744167451674616747167481674916750167511675216753167541675516756167571675816759167601676116762167631676416765167661676716768167691677016771167721677316774167751677616777167781677916780167811678216783167841678516786167871678816789167901679116792167931679416795167961679716798167991680016801168021680316804168051680616807168081680916810168111681216813168141681516816168171681816819168201682116822168231682416825168261682716828168291683016831168321683316834168351683616837168381683916840168411684216843168441684516846168471684816849168501685116852168531685416855168561685716858168591686016861168621686316864168651686616867168681686916870168711687216873168741687516876168771687816879168801688116882168831688416885168861688716888168891689016891168921689316894168951689616897168981689916900169011690216903169041690516906169071690816909169101691116912169131691416915169161691716918169191692016921169221692316924169251692616927169281692916930169311693216933169341693516936169371693816939169401694116942169431694416945169461694716948169491695016951169521695316954169551695616957169581695916960169611696216963169641696516966169671696816969169701697116972169731697416975169761697716978169791698016981169821698316984169851698616987169881698916990169911699216993169941699516996169971699816999170001700117002170031700417005170061700717008170091701017011170121701317014170151701617017170181701917020170211702217023170241702517026170271702817029170301703117032170331703417035170361703717038170391704017041170421704317044170451704617047170481704917050170511705217053170541705517056170571705817059170601706117062170631706417065170661706717068170691707017071170721707317074170751707617077170781707917080170811708217083170841708517086170871708817089170901709117092170931709417095170961709717098170991710017101171021710317104171051710617107171081710917110171111711217113171141711517116171171711817119171201712117122171231712417125171261712717128171291713017131171321713317134171351713617137171381713917140171411714217143171441714517146171471714817149171501715117152171531715417155171561715717158171591716017161171621716317164171651716617167171681716917170171711717217173171741717517176171771717817179171801718117182171831718417185171861718717188171891719017191171921719317194171951719617197171981719917200172011720217203172041720517206172071720817209172101721117212172131721417215172161721717218172191722017221172221722317224172251722617227172281722917230172311723217233172341723517236172371723817239172401724117242172431724417245172461724717248172491725017251172521725317254172551725617257172581725917260172611726217263172641726517266172671726817269172701727117272172731727417275172761727717278172791728017281172821728317284172851728617287172881728917290172911729217293172941729517296172971729817299173001730117302173031730417305173061730717308173091731017311173121731317314173151731617317173181731917320173211732217323173241732517326173271732817329173301733117332173331733417335173361733717338173391734017341173421734317344173451734617347173481734917350173511735217353173541735517356173571735817359173601736117362173631736417365173661736717368173691737017371173721737317374173751737617377173781737917380173811738217383173841738517386173871738817389173901739117392173931739417395173961739717398173991740017401174021740317404174051740617407174081740917410174111741217413174141741517416174171741817419174201742117422174231742417425174261742717428174291743017431174321743317434174351743617437174381743917440174411744217443174441744517446174471744817449174501745117452174531745417455174561745717458174591746017461174621746317464174651746617467174681746917470174711747217473174741747517476174771747817479174801748117482174831748417485174861748717488174891749017491174921749317494174951749617497174981749917500175011750217503175041750517506175071750817509175101751117512175131751417515175161751717518175191752017521175221752317524175251752617527175281752917530175311753217533175341753517536175371753817539175401754117542175431754417545175461754717548175491755017551175521755317554175551755617557175581755917560175611756217563175641756517566175671756817569175701757117572175731757417575175761757717578175791758017581175821758317584175851758617587175881758917590175911759217593175941759517596175971759817599176001760117602176031760417605176061760717608176091761017611176121761317614176151761617617176181761917620176211762217623176241762517626176271762817629176301763117632176331763417635176361763717638176391764017641176421764317644176451764617647176481764917650176511765217653176541765517656176571765817659176601766117662176631766417665176661766717668176691767017671176721767317674176751767617677176781767917680176811768217683176841768517686176871768817689176901769117692176931769417695176961769717698176991770017701177021770317704177051770617707177081770917710177111771217713177141771517716177171771817719177201772117722177231772417725177261772717728177291773017731177321773317734177351773617737177381773917740177411774217743177441774517746177471774817749177501775117752177531775417755177561775717758177591776017761177621776317764177651776617767177681776917770177711777217773177741777517776177771777817779177801778117782177831778417785177861778717788177891779017791177921779317794177951779617797177981779917800178011780217803178041780517806178071780817809178101781117812178131781417815178161781717818178191782017821178221782317824178251782617827178281782917830178311783217833178341783517836178371783817839178401784117842178431784417845178461784717848178491785017851178521785317854178551785617857178581785917860178611786217863178641786517866178671786817869178701787117872178731787417875178761787717878178791788017881178821788317884178851788617887178881788917890178911789217893178941789517896178971789817899179001790117902179031790417905179061790717908179091791017911179121791317914179151791617917179181791917920179211792217923179241792517926179271792817929179301793117932179331793417935179361793717938179391794017941179421794317944179451794617947179481794917950179511795217953179541795517956179571795817959179601796117962179631796417965179661796717968179691797017971179721797317974179751797617977179781797917980179811798217983179841798517986179871798817989179901799117992179931799417995179961799717998179991800018001180021800318004180051800618007180081800918010180111801218013180141801518016180171801818019180201802118022180231802418025180261802718028180291803018031180321803318034180351803618037180381803918040180411804218043180441804518046180471804818049180501805118052180531805418055180561805718058180591806018061180621806318064180651806618067180681806918070180711807218073180741807518076180771807818079180801808118082180831808418085180861808718088180891809018091180921809318094180951809618097180981809918100181011810218103181041810518106181071810818109181101811118112181131811418115181161811718118181191812018121181221812318124181251812618127181281812918130181311813218133181341813518136181371813818139181401814118142181431814418145181461814718148181491815018151181521815318154181551815618157181581815918160181611816218163181641816518166181671816818169181701817118172181731817418175181761817718178181791818018181181821818318184181851818618187181881818918190181911819218193181941819518196181971819818199182001820118202182031820418205182061820718208182091821018211182121821318214182151821618217182181821918220182211822218223182241822518226182271822818229182301823118232182331823418235182361823718238182391824018241182421824318244182451824618247182481824918250182511825218253182541825518256182571825818259182601826118262182631826418265182661826718268182691827018271182721827318274182751827618277182781827918280182811828218283182841828518286182871828818289182901829118292182931829418295182961829718298182991830018301183021830318304183051830618307183081830918310183111831218313183141831518316183171831818319183201832118322183231832418325183261832718328183291833018331183321833318334183351833618337183381833918340183411834218343183441834518346183471834818349183501835118352183531835418355183561835718358183591836018361183621836318364183651836618367183681836918370183711837218373183741837518376183771837818379183801838118382183831838418385183861838718388183891839018391183921839318394183951839618397183981839918400184011840218403184041840518406184071840818409184101841118412184131841418415184161841718418184191842018421184221842318424184251842618427184281842918430184311843218433184341843518436184371843818439184401844118442184431844418445184461844718448184491845018451184521845318454184551845618457184581845918460184611846218463184641846518466184671846818469184701847118472184731847418475184761847718478184791848018481184821848318484184851848618487184881848918490184911849218493184941849518496184971849818499185001850118502185031850418505185061850718508185091851018511185121851318514185151851618517185181851918520185211852218523185241852518526185271852818529185301853118532185331853418535185361853718538185391854018541185421854318544185451854618547185481854918550185511855218553185541855518556185571855818559185601856118562185631856418565185661856718568185691857018571185721857318574185751857618577185781857918580185811858218583185841858518586185871858818589185901859118592185931859418595185961859718598185991860018601186021860318604186051860618607186081860918610186111861218613186141861518616186171861818619186201862118622186231862418625186261862718628186291863018631186321863318634186351863618637186381863918640186411864218643186441864518646186471864818649186501865118652186531865418655186561865718658186591866018661186621866318664186651866618667186681866918670186711867218673186741867518676186771867818679186801868118682186831868418685186861868718688186891869018691186921869318694186951869618697186981869918700187011870218703187041870518706187071870818709187101871118712187131871418715187161871718718187191872018721187221872318724187251872618727187281872918730187311873218733187341873518736187371873818739187401874118742187431874418745187461874718748187491875018751187521875318754187551875618757187581875918760187611876218763187641876518766187671876818769187701877118772187731877418775187761877718778187791878018781187821878318784187851878618787187881878918790187911879218793187941879518796187971879818799188001880118802188031880418805188061880718808188091881018811188121881318814188151881618817188181881918820188211882218823188241882518826188271882818829188301883118832188331883418835188361883718838188391884018841188421884318844188451884618847188481884918850188511885218853188541885518856188571885818859188601886118862188631886418865188661886718868188691887018871188721887318874188751887618877188781887918880188811888218883188841888518886188871888818889188901889118892188931889418895188961889718898188991890018901189021890318904189051890618907189081890918910189111891218913189141891518916189171891818919189201892118922189231892418925189261892718928189291893018931189321893318934189351893618937189381893918940189411894218943189441894518946189471894818949189501895118952189531895418955189561895718958189591896018961189621896318964189651896618967189681896918970189711897218973189741897518976189771897818979189801898118982189831898418985189861898718988189891899018991189921899318994189951899618997189981899919000190011900219003190041900519006190071900819009190101901119012190131901419015190161901719018190191902019021190221902319024190251902619027190281902919030190311903219033190341903519036190371903819039190401904119042190431904419045190461904719048190491905019051190521905319054190551905619057190581905919060190611906219063190641906519066190671906819069190701907119072190731907419075190761907719078190791908019081190821908319084190851908619087190881908919090190911909219093190941909519096190971909819099191001910119102191031910419105191061910719108191091911019111191121911319114191151911619117191181911919120191211912219123191241912519126191271912819129191301913119132191331913419135191361913719138191391914019141191421914319144191451914619147191481914919150191511915219153191541915519156191571915819159191601916119162191631916419165191661916719168191691917019171191721917319174191751917619177191781917919180191811918219183191841918519186191871918819189191901919119192191931919419195191961919719198191991920019201192021920319204192051920619207192081920919210192111921219213192141921519216192171921819219192201922119222192231922419225192261922719228192291923019231192321923319234192351923619237192381923919240192411924219243192441924519246192471924819249192501925119252192531925419255192561925719258192591926019261192621926319264192651926619267192681926919270192711927219273192741927519276192771927819279192801928119282192831928419285192861928719288192891929019291192921929319294192951929619297192981929919300193011930219303193041930519306193071930819309193101931119312193131931419315193161931719318193191932019321193221932319324193251932619327193281932919330193311933219333193341933519336193371933819339193401934119342193431934419345193461934719348193491935019351193521935319354193551935619357193581935919360193611936219363193641936519366193671936819369193701937119372193731937419375193761937719378193791938019381193821938319384193851938619387193881938919390193911939219393193941939519396193971939819399194001940119402194031940419405194061940719408194091941019411194121941319414194151941619417194181941919420194211942219423194241942519426194271942819429194301943119432194331943419435194361943719438194391944019441194421944319444194451944619447194481944919450194511945219453194541945519456194571945819459194601946119462194631946419465194661946719468194691947019471194721947319474194751947619477194781947919480194811948219483194841948519486194871948819489194901949119492194931949419495194961949719498194991950019501195021950319504195051950619507195081950919510195111951219513195141951519516195171951819519195201952119522195231952419525195261952719528195291953019531195321953319534195351953619537195381953919540195411954219543195441954519546195471954819549195501955119552195531955419555195561955719558195591956019561195621956319564195651956619567195681956919570195711957219573195741957519576195771957819579195801958119582195831958419585195861958719588195891959019591195921959319594195951959619597195981959919600196011960219603196041960519606196071960819609196101961119612196131961419615196161961719618196191962019621196221962319624196251962619627196281962919630196311963219633196341963519636196371963819639196401964119642196431964419645196461964719648196491965019651196521965319654196551965619657196581965919660196611966219663196641966519666196671966819669196701967119672196731967419675196761967719678196791968019681196821968319684196851968619687196881968919690196911969219693196941969519696196971969819699197001970119702197031970419705197061970719708197091971019711197121971319714197151971619717197181971919720197211972219723197241972519726197271972819729197301973119732197331973419735197361973719738197391974019741197421974319744197451974619747197481974919750197511975219753197541975519756197571975819759197601976119762197631976419765197661976719768197691977019771197721977319774197751977619777197781977919780197811978219783197841978519786197871978819789197901979119792197931979419795197961979719798197991980019801198021980319804198051980619807198081980919810198111981219813198141981519816198171981819819198201982119822198231982419825198261982719828198291983019831198321983319834198351983619837198381983919840198411984219843198441984519846198471984819849198501985119852198531985419855198561985719858198591986019861198621986319864198651986619867198681986919870198711987219873198741987519876198771987819879198801988119882198831988419885198861988719888198891989019891198921989319894198951989619897198981989919900199011990219903199041990519906199071990819909199101991119912199131991419915199161991719918199191992019921199221992319924199251992619927199281992919930199311993219933199341993519936199371993819939199401994119942199431994419945199461994719948199491995019951199521995319954199551995619957199581995919960199611996219963199641996519966199671996819969199701997119972199731997419975199761997719978199791998019981199821998319984199851998619987199881998919990199911999219993199941999519996199971999819999200002000120002200032000420005200062000720008200092001020011200122001320014200152001620017200182001920020200212002220023200242002520026200272002820029200302003120032200332003420035200362003720038200392004020041200422004320044200452004620047200482004920050200512005220053200542005520056200572005820059200602006120062200632006420065200662006720068200692007020071200722007320074200752007620077200782007920080200812008220083200842008520086200872008820089200902009120092200932009420095200962009720098200992010020101201022010320104201052010620107201082010920110201112011220113201142011520116201172011820119201202012120122201232012420125201262012720128201292013020131201322013320134201352013620137201382013920140201412014220143201442014520146201472014820149201502015120152201532015420155201562015720158201592016020161201622016320164201652016620167201682016920170201712017220173201742017520176201772017820179201802018120182201832018420185201862018720188201892019020191201922019320194201952019620197201982019920200202012020220203202042020520206202072020820209202102021120212202132021420215202162021720218202192022020221202222022320224202252022620227202282022920230202312023220233202342023520236202372023820239202402024120242202432024420245202462024720248202492025020251202522025320254202552025620257202582025920260202612026220263202642026520266202672026820269202702027120272202732027420275202762027720278202792028020281202822028320284202852028620287202882028920290202912029220293202942029520296202972029820299203002030120302203032030420305203062030720308203092031020311203122031320314203152031620317203182031920320203212032220323203242032520326203272032820329203302033120332203332033420335203362033720338203392034020341203422034320344203452034620347203482034920350203512035220353203542035520356203572035820359203602036120362203632036420365203662036720368203692037020371203722037320374203752037620377203782037920380203812038220383203842038520386203872038820389203902039120392203932039420395203962039720398203992040020401204022040320404204052040620407204082040920410204112041220413204142041520416204172041820419204202042120422204232042420425204262042720428204292043020431204322043320434204352043620437204382043920440204412044220443204442044520446204472044820449204502045120452204532045420455204562045720458204592046020461204622046320464204652046620467204682046920470204712047220473204742047520476204772047820479204802048120482204832048420485204862048720488204892049020491204922049320494204952049620497204982049920500205012050220503205042050520506205072050820509205102051120512205132051420515205162051720518205192052020521205222052320524205252052620527205282052920530205312053220533205342053520536205372053820539205402054120542205432054420545205462054720548205492055020551205522055320554205552055620557205582055920560205612056220563205642056520566205672056820569205702057120572205732057420575205762057720578205792058020581205822058320584205852058620587205882058920590205912059220593205942059520596205972059820599206002060120602206032060420605206062060720608206092061020611206122061320614206152061620617206182061920620206212062220623206242062520626206272062820629206302063120632206332063420635206362063720638206392064020641206422064320644206452064620647206482064920650206512065220653206542065520656206572065820659206602066120662206632066420665206662066720668206692067020671206722067320674206752067620677206782067920680206812068220683206842068520686206872068820689206902069120692206932069420695206962069720698206992070020701207022070320704207052070620707207082070920710207112071220713207142071520716207172071820719207202072120722207232072420725207262072720728207292073020731207322073320734207352073620737207382073920740207412074220743207442074520746207472074820749207502075120752207532075420755207562075720758207592076020761207622076320764207652076620767207682076920770207712077220773207742077520776207772077820779207802078120782207832078420785207862078720788207892079020791207922079320794207952079620797207982079920800208012080220803208042080520806208072080820809208102081120812208132081420815208162081720818208192082020821208222082320824208252082620827208282082920830208312083220833208342083520836208372083820839208402084120842208432084420845208462084720848208492085020851208522085320854208552085620857208582085920860208612086220863208642086520866208672086820869208702087120872208732087420875208762087720878208792088020881208822088320884208852088620887208882088920890208912089220893208942089520896208972089820899209002090120902209032090420905209062090720908209092091020911209122091320914209152091620917209182091920920209212092220923209242092520926209272092820929209302093120932209332093420935209362093720938209392094020941209422094320944209452094620947209482094920950209512095220953209542095520956209572095820959209602096120962209632096420965209662096720968209692097020971209722097320974209752097620977209782097920980209812098220983209842098520986209872098820989209902099120992209932099420995209962099720998209992100021001210022100321004210052100621007210082100921010210112101221013210142101521016210172101821019210202102121022210232102421025210262102721028210292103021031210322103321034210352103621037210382103921040210412104221043210442104521046210472104821049210502105121052210532105421055210562105721058210592106021061210622106321064210652106621067210682106921070210712107221073210742107521076210772107821079210802108121082210832108421085210862108721088210892109021091210922109321094210952109621097210982109921100211012110221103211042110521106211072110821109211102111121112211132111421115211162111721118211192112021121211222112321124211252112621127211282112921130211312113221133211342113521136211372113821139211402114121142211432114421145211462114721148211492115021151211522115321154211552115621157211582115921160211612116221163211642116521166211672116821169211702117121172211732117421175211762117721178211792118021181211822118321184211852118621187211882118921190211912119221193211942119521196211972119821199212002120121202212032120421205212062120721208212092121021211212122121321214212152121621217212182121921220212212122221223212242122521226212272122821229212302123121232212332123421235212362123721238212392124021241212422124321244212452124621247212482124921250212512125221253212542125521256212572125821259212602126121262212632126421265212662126721268212692127021271212722127321274212752127621277212782127921280212812128221283212842128521286212872128821289212902129121292212932129421295212962129721298212992130021301213022130321304213052130621307213082130921310213112131221313213142131521316213172131821319213202132121322213232132421325213262132721328213292133021331213322133321334213352133621337213382133921340213412134221343213442134521346213472134821349213502135121352213532135421355213562135721358213592136021361213622136321364213652136621367213682136921370213712137221373213742137521376213772137821379213802138121382213832138421385213862138721388213892139021391213922139321394213952139621397213982139921400214012140221403214042140521406214072140821409214102141121412214132141421415214162141721418214192142021421214222142321424214252142621427214282142921430214312143221433214342143521436214372143821439214402144121442214432144421445214462144721448214492145021451214522145321454214552145621457214582145921460214612146221463214642146521466214672146821469214702147121472214732147421475214762147721478214792148021481214822148321484214852148621487214882148921490214912149221493214942149521496214972149821499215002150121502215032150421505215062150721508215092151021511215122151321514215152151621517215182151921520215212152221523215242152521526215272152821529215302153121532215332153421535215362153721538215392154021541215422154321544215452154621547215482154921550215512155221553215542155521556215572155821559215602156121562215632156421565215662156721568215692157021571215722157321574215752157621577215782157921580215812158221583215842158521586215872158821589215902159121592215932159421595215962159721598215992160021601216022160321604216052160621607216082160921610216112161221613216142161521616216172161821619216202162121622216232162421625216262162721628216292163021631216322163321634216352163621637216382163921640216412164221643216442164521646216472164821649216502165121652216532165421655216562165721658216592166021661216622166321664216652166621667216682166921670216712167221673216742167521676216772167821679216802168121682216832168421685216862168721688216892169021691216922169321694216952169621697216982169921700217012170221703217042170521706217072170821709217102171121712217132171421715217162171721718217192172021721217222172321724217252172621727217282172921730217312173221733217342173521736217372173821739217402174121742217432174421745217462174721748217492175021751217522175321754217552175621757217582175921760217612176221763217642176521766217672176821769217702177121772217732177421775217762177721778217792178021781217822178321784217852178621787217882178921790217912179221793217942179521796217972179821799218002180121802218032180421805218062180721808218092181021811218122181321814218152181621817218182181921820218212182221823218242182521826218272182821829218302183121832218332183421835218362183721838218392184021841218422184321844218452184621847218482184921850218512185221853218542185521856218572185821859218602186121862218632186421865218662186721868218692187021871218722187321874218752187621877218782187921880218812188221883218842188521886218872188821889218902189121892218932189421895218962189721898218992190021901219022190321904219052190621907219082190921910219112191221913219142191521916219172191821919219202192121922219232192421925219262192721928219292193021931219322193321934219352193621937219382193921940219412194221943219442194521946219472194821949219502195121952219532195421955219562195721958219592196021961219622196321964219652196621967219682196921970219712197221973219742197521976219772197821979219802198121982219832198421985219862198721988219892199021991219922199321994219952199621997219982199922000220012200222003220042200522006220072200822009220102201122012220132201422015220162201722018220192202022021220222202322024220252202622027220282202922030220312203222033220342203522036220372203822039220402204122042220432204422045220462204722048220492205022051220522205322054220552205622057220582205922060220612206222063220642206522066220672206822069220702207122072220732207422075220762207722078220792208022081220822208322084220852208622087220882208922090220912209222093220942209522096220972209822099221002210122102221032210422105221062210722108221092211022111221122211322114221152211622117221182211922120221212212222123221242212522126221272212822129221302213122132221332213422135221362213722138221392214022141221422214322144221452214622147221482214922150221512215222153221542215522156221572215822159221602216122162221632216422165221662216722168221692217022171221722217322174221752217622177221782217922180221812218222183221842218522186221872218822189221902219122192221932219422195221962219722198221992220022201222022220322204222052220622207222082220922210222112221222213222142221522216222172221822219222202222122222222232222422225222262222722228222292223022231222322223322234222352223622237222382223922240222412224222243222442224522246222472224822249222502225122252222532225422255222562225722258222592226022261222622226322264222652226622267222682226922270222712227222273222742227522276222772227822279222802228122282222832228422285222862228722288222892229022291222922229322294222952229622297222982229922300223012230222303223042230522306223072230822309223102231122312223132231422315223162231722318223192232022321223222232322324223252232622327223282232922330223312233222333223342233522336223372233822339223402234122342223432234422345223462234722348223492235022351223522235322354223552235622357223582235922360223612236222363223642236522366223672236822369223702237122372223732237422375223762237722378223792238022381223822238322384223852238622387223882238922390223912239222393223942239522396223972239822399224002240122402224032240422405224062240722408224092241022411224122241322414224152241622417224182241922420224212242222423224242242522426224272242822429224302243122432224332243422435224362243722438224392244022441224422244322444224452244622447224482244922450224512245222453224542245522456224572245822459224602246122462224632246422465224662246722468224692247022471224722247322474224752247622477224782247922480224812248222483224842248522486224872248822489224902249122492224932249422495224962249722498224992250022501225022250322504225052250622507225082250922510225112251222513225142251522516225172251822519225202252122522225232252422525225262252722528225292253022531225322253322534225352253622537225382253922540225412254222543225442254522546225472254822549225502255122552225532255422555225562255722558225592256022561225622256322564225652256622567225682256922570225712257222573225742257522576225772257822579225802258122582225832258422585225862258722588225892259022591225922259322594225952259622597225982259922600226012260222603226042260522606226072260822609226102261122612226132261422615226162261722618226192262022621226222262322624226252262622627226282262922630226312263222633226342263522636226372263822639226402264122642226432264422645226462264722648226492265022651226522265322654226552265622657226582265922660226612266222663226642266522666226672266822669226702267122672226732267422675226762267722678226792268022681226822268322684226852268622687226882268922690226912269222693226942269522696226972269822699227002270122702227032270422705227062270722708227092271022711227122271322714227152271622717227182271922720227212272222723227242272522726227272272822729227302273122732227332273422735227362273722738227392274022741227422274322744227452274622747227482274922750227512275222753227542275522756227572275822759227602276122762227632276422765227662276722768227692277022771227722277322774227752277622777227782277922780227812278222783227842278522786227872278822789227902279122792227932279422795227962279722798227992280022801228022280322804228052280622807228082280922810228112281222813228142281522816228172281822819228202282122822228232282422825228262282722828228292283022831228322283322834228352283622837228382283922840228412284222843228442284522846228472284822849228502285122852228532285422855228562285722858228592286022861228622286322864228652286622867228682286922870228712287222873228742287522876228772287822879228802288122882228832288422885228862288722888228892289022891228922289322894228952289622897228982289922900229012290222903229042290522906229072290822909229102291122912229132291422915229162291722918229192292022921229222292322924229252292622927229282292922930229312293222933229342293522936229372293822939229402294122942229432294422945229462294722948229492295022951229522295322954229552295622957229582295922960229612296222963229642296522966229672296822969229702297122972229732297422975229762297722978229792298022981229822298322984229852298622987229882298922990229912299222993229942299522996229972299822999230002300123002230032300423005230062300723008230092301023011230122301323014230152301623017230182301923020230212302223023230242302523026230272302823029230302303123032230332303423035230362303723038230392304023041230422304323044230452304623047230482304923050230512305223053230542305523056230572305823059230602306123062230632306423065230662306723068230692307023071230722307323074230752307623077230782307923080230812308223083230842308523086230872308823089230902309123092230932309423095230962309723098230992310023101231022310323104231052310623107231082310923110231112311223113231142311523116231172311823119231202312123122231232312423125231262312723128231292313023131231322313323134231352313623137231382313923140231412314223143231442314523146231472314823149231502315123152231532315423155231562315723158231592316023161231622316323164231652316623167231682316923170231712317223173231742317523176231772317823179231802318123182231832318423185231862318723188231892319023191231922319323194231952319623197231982319923200232012320223203232042320523206232072320823209232102321123212232132321423215232162321723218232192322023221232222322323224232252322623227232282322923230232312323223233232342323523236232372323823239232402324123242232432324423245232462324723248232492325023251232522325323254232552325623257232582325923260232612326223263232642326523266232672326823269232702327123272232732327423275232762327723278232792328023281232822328323284232852328623287232882328923290232912329223293232942329523296232972329823299233002330123302233032330423305233062330723308233092331023311233122331323314233152331623317233182331923320233212332223323233242332523326233272332823329233302333123332233332333423335233362333723338233392334023341233422334323344233452334623347233482334923350233512335223353233542335523356233572335823359233602336123362233632336423365233662336723368233692337023371233722337323374233752337623377233782337923380233812338223383233842338523386233872338823389233902339123392233932339423395233962339723398233992340023401234022340323404234052340623407234082340923410234112341223413234142341523416234172341823419234202342123422234232342423425234262342723428234292343023431234322343323434234352343623437234382343923440234412344223443234442344523446234472344823449234502345123452234532345423455234562345723458234592346023461234622346323464234652346623467234682346923470234712347223473234742347523476234772347823479234802348123482234832348423485234862348723488234892349023491234922349323494234952349623497234982349923500235012350223503235042350523506235072350823509235102351123512235132351423515235162351723518235192352023521235222352323524235252352623527235282352923530235312353223533235342353523536235372353823539235402354123542235432354423545235462354723548235492355023551235522355323554235552355623557235582355923560235612356223563235642356523566235672356823569235702357123572235732357423575235762357723578235792358023581235822358323584235852358623587235882358923590235912359223593235942359523596235972359823599236002360123602236032360423605236062360723608236092361023611236122361323614236152361623617236182361923620236212362223623236242362523626236272362823629236302363123632236332363423635236362363723638236392364023641236422364323644236452364623647236482364923650236512365223653236542365523656236572365823659236602366123662236632366423665236662366723668236692367023671236722367323674236752367623677236782367923680236812368223683236842368523686236872368823689236902369123692236932369423695236962369723698236992370023701237022370323704237052370623707237082370923710237112371223713237142371523716237172371823719237202372123722237232372423725237262372723728237292373023731237322373323734237352373623737237382373923740237412374223743237442374523746237472374823749237502375123752237532375423755237562375723758237592376023761237622376323764237652376623767237682376923770237712377223773237742377523776237772377823779237802378123782237832378423785237862378723788237892379023791237922379323794237952379623797237982379923800238012380223803238042380523806238072380823809238102381123812238132381423815238162381723818238192382023821238222382323824238252382623827238282382923830238312383223833238342383523836238372383823839238402384123842238432384423845238462384723848238492385023851238522385323854238552385623857238582385923860238612386223863238642386523866238672386823869238702387123872238732387423875238762387723878238792388023881238822388323884238852388623887238882388923890238912389223893238942389523896238972389823899239002390123902239032390423905239062390723908239092391023911239122391323914239152391623917239182391923920239212392223923239242392523926239272392823929239302393123932239332393423935239362393723938239392394023941239422394323944239452394623947239482394923950239512395223953239542395523956239572395823959239602396123962239632396423965239662396723968239692397023971239722397323974239752397623977239782397923980239812398223983239842398523986239872398823989239902399123992239932399423995239962399723998239992400024001240022400324004240052400624007240082400924010240112401224013240142401524016240172401824019240202402124022240232402424025240262402724028240292403024031240322403324034240352403624037240382403924040240412404224043240442404524046240472404824049240502405124052240532405424055240562405724058240592406024061240622406324064240652406624067240682406924070240712407224073240742407524076240772407824079240802408124082240832408424085240862408724088240892409024091240922409324094240952409624097240982409924100241012410224103241042410524106241072410824109241102411124112241132411424115241162411724118241192412024121241222412324124241252412624127241282412924130241312413224133241342413524136241372413824139241402414124142241432414424145241462414724148241492415024151241522415324154241552415624157241582415924160241612416224163241642416524166241672416824169241702417124172241732417424175241762417724178241792418024181241822418324184241852418624187241882418924190241912419224193241942419524196241972419824199242002420124202242032420424205242062420724208242092421024211242122421324214242152421624217242182421924220242212422224223242242422524226242272422824229242302423124232242332423424235242362423724238242392424024241242422424324244242452424624247242482424924250242512425224253242542425524256242572425824259242602426124262242632426424265242662426724268242692427024271242722427324274242752427624277242782427924280242812428224283242842428524286242872428824289242902429124292242932429424295242962429724298242992430024301243022430324304243052430624307243082430924310243112431224313243142431524316243172431824319243202432124322243232432424325243262432724328243292433024331243322433324334243352433624337243382433924340243412434224343243442434524346243472434824349243502435124352243532435424355243562435724358243592436024361243622436324364243652436624367243682436924370243712437224373243742437524376243772437824379243802438124382243832438424385243862438724388243892439024391243922439324394243952439624397243982439924400244012440224403244042440524406244072440824409244102441124412244132441424415244162441724418244192442024421244222442324424244252442624427244282442924430244312443224433244342443524436244372443824439244402444124442244432444424445244462444724448244492445024451244522445324454244552445624457244582445924460244612446224463244642446524466244672446824469244702447124472244732447424475244762447724478244792448024481244822448324484244852448624487244882448924490244912449224493244942449524496244972449824499245002450124502245032450424505245062450724508245092451024511245122451324514245152451624517245182451924520245212452224523245242452524526245272452824529245302453124532245332453424535245362453724538245392454024541245422454324544245452454624547245482454924550245512455224553245542455524556245572455824559245602456124562245632456424565245662456724568245692457024571245722457324574245752457624577245782457924580245812458224583245842458524586245872458824589245902459124592245932459424595245962459724598245992460024601246022460324604246052460624607246082460924610246112461224613246142461524616246172461824619246202462124622246232462424625246262462724628246292463024631246322463324634246352463624637246382463924640246412464224643246442464524646246472464824649246502465124652246532465424655246562465724658246592466024661246622466324664246652466624667246682466924670246712467224673246742467524676246772467824679246802468124682246832468424685246862468724688246892469024691246922469324694246952469624697246982469924700247012470224703247042470524706247072470824709247102471124712247132471424715247162471724718247192472024721247222472324724247252472624727247282472924730247312473224733247342473524736247372473824739247402474124742247432474424745247462474724748247492475024751247522475324754247552475624757247582475924760247612476224763247642476524766247672476824769247702477124772247732477424775247762477724778247792478024781247822478324784247852478624787247882478924790247912479224793247942479524796247972479824799248002480124802248032480424805248062480724808248092481024811248122481324814248152481624817248182481924820248212482224823248242482524826248272482824829248302483124832248332483424835248362483724838248392484024841248422484324844248452484624847248482484924850248512485224853248542485524856248572485824859248602486124862248632486424865248662486724868248692487024871248722487324874248752487624877248782487924880248812488224883248842488524886248872488824889248902489124892248932489424895248962489724898248992490024901249022490324904249052490624907249082490924910249112491224913249142491524916249172491824919249202492124922249232492424925249262492724928249292493024931249322493324934249352493624937249382493924940249412494224943249442494524946249472494824949249502495124952249532495424955249562495724958249592496024961249622496324964249652496624967249682496924970249712497224973249742497524976249772497824979249802498124982249832498424985249862498724988249892499024991249922499324994249952499624997249982499925000250012500225003250042500525006250072500825009250102501125012250132501425015250162501725018250192502025021250222502325024250252502625027250282502925030250312503225033250342503525036250372503825039250402504125042250432504425045250462504725048250492505025051250522505325054250552505625057250582505925060250612506225063250642506525066250672506825069250702507125072250732507425075250762507725078250792508025081250822508325084250852508625087250882508925090250912509225093250942509525096250972509825099251002510125102251032510425105251062510725108251092511025111251122511325114251152511625117251182511925120251212512225123251242512525126251272512825129251302513125132251332513425135251362513725138251392514025141251422514325144251452514625147251482514925150251512515225153251542515525156251572515825159251602516125162251632516425165251662516725168251692517025171251722517325174251752517625177251782517925180251812518225183251842518525186251872518825189251902519125192251932519425195251962519725198251992520025201252022520325204252052520625207252082520925210252112521225213252142521525216252172521825219252202522125222252232522425225252262522725228252292523025231252322523325234252352523625237252382523925240252412524225243252442524525246252472524825249252502525125252252532525425255252562525725258252592526025261252622526325264252652526625267252682526925270252712527225273252742527525276252772527825279252802528125282252832528425285252862528725288252892529025291252922529325294252952529625297252982529925300253012530225303253042530525306253072530825309253102531125312253132531425315253162531725318253192532025321253222532325324253252532625327253282532925330253312533225333253342533525336253372533825339253402534125342253432534425345253462534725348253492535025351253522535325354253552535625357253582535925360253612536225363253642536525366253672536825369253702537125372253732537425375253762537725378253792538025381253822538325384253852538625387253882538925390253912539225393253942539525396253972539825399254002540125402254032540425405254062540725408254092541025411254122541325414254152541625417254182541925420254212542225423254242542525426254272542825429254302543125432254332543425435254362543725438254392544025441254422544325444254452544625447254482544925450254512545225453254542545525456254572545825459254602546125462254632546425465254662546725468254692547025471254722547325474254752547625477254782547925480254812548225483254842548525486254872548825489254902549125492254932549425495254962549725498254992550025501255022550325504255052550625507255082550925510255112551225513255142551525516255172551825519255202552125522255232552425525255262552725528255292553025531255322553325534255352553625537255382553925540255412554225543255442554525546255472554825549255502555125552255532555425555255562555725558255592556025561255622556325564255652556625567255682556925570255712557225573255742557525576255772557825579255802558125582255832558425585255862558725588255892559025591255922559325594255952559625597255982559925600256012560225603256042560525606256072560825609256102561125612256132561425615256162561725618256192562025621256222562325624256252562625627256282562925630256312563225633256342563525636256372563825639256402564125642256432564425645256462564725648256492565025651256522565325654256552565625657256582565925660256612566225663256642566525666256672566825669256702567125672256732567425675256762567725678256792568025681256822568325684256852568625687256882568925690256912569225693256942569525696256972569825699257002570125702257032570425705257062570725708257092571025711257122571325714257152571625717257182571925720257212572225723257242572525726257272572825729257302573125732257332573425735257362573725738257392574025741257422574325744257452574625747257482574925750257512575225753257542575525756257572575825759257602576125762257632576425765257662576725768257692577025771257722577325774257752577625777257782577925780257812578225783257842578525786257872578825789257902579125792257932579425795257962579725798257992580025801258022580325804258052580625807258082580925810258112581225813258142581525816258172581825819258202582125822258232582425825258262582725828258292583025831258322583325834258352583625837258382583925840258412584225843258442584525846258472584825849258502585125852258532585425855258562585725858258592586025861258622586325864258652586625867258682586925870258712587225873258742587525876258772587825879258802588125882258832588425885258862588725888258892589025891258922589325894258952589625897258982589925900259012590225903259042590525906259072590825909259102591125912259132591425915259162591725918259192592025921259222592325924259252592625927259282592925930259312593225933259342593525936259372593825939259402594125942259432594425945259462594725948259492595025951259522595325954259552595625957259582595925960259612596225963259642596525966259672596825969259702597125972259732597425975259762597725978259792598025981259822598325984259852598625987259882598925990259912599225993259942599525996259972599825999260002600126002260032600426005260062600726008260092601026011260122601326014260152601626017260182601926020260212602226023260242602526026260272602826029260302603126032260332603426035260362603726038260392604026041260422604326044260452604626047
  1. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/Documentation/sysrq.txt linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/Documentation/sysrq.txt
  2. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/Documentation/sysrq.txt 2017-04-16 10:37:28.000000000 +0200
  3. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/Documentation/sysrq.txt 2017-04-18 17:54:21.000000000 +0200
  4. @@ -59,10 +59,17 @@
  5. On other - If you know of the key combos for other architectures, please
  6. let me know so I can add them to this section.
  7. -On all - write a character to /proc/sysrq-trigger. e.g.:
  8. -
  9. +On all - write a character to /proc/sysrq-trigger, e.g.:
  10. echo t > /proc/sysrq-trigger
  11. +On all - Enable network SysRq by writing a cookie to icmp_echo_sysrq, e.g.
  12. + echo 0x01020304 >/proc/sys/net/ipv4/icmp_echo_sysrq
  13. + Send an ICMP echo request with this pattern plus the particular
  14. + SysRq command key. Example:
  15. + # ping -c1 -s57 -p0102030468
  16. + will trigger the SysRq-H (help) command.
  17. +
  18. +
  19. * What are the 'command' keys?
  20. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  21. 'b' - Will immediately reboot the system without syncing or unmounting
  22. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/Documentation/trace/histograms.txt linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/Documentation/trace/histograms.txt
  23. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/Documentation/trace/histograms.txt 1970-01-01 01:00:00.000000000 +0100
  24. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/Documentation/trace/histograms.txt 2017-04-18 17:54:22.000000000 +0200
  25. @@ -0,0 +1,186 @@
  26. + Using the Linux Kernel Latency Histograms
  27. +
  28. +
  29. +This document gives a short explanation how to enable, configure and use
  30. +latency histograms. Latency histograms are primarily relevant in the
  31. +context of real-time enabled kernels (CONFIG_PREEMPT/CONFIG_PREEMPT_RT)
  32. +and are used in the quality management of the Linux real-time
  33. +capabilities.
  34. +
  35. +
  36. +* Purpose of latency histograms
  37. +
  38. +A latency histogram continuously accumulates the frequencies of latency
  39. +data. There are two types of histograms
  40. +- potential sources of latencies
  41. +- effective latencies
  42. +
  43. +
  44. +* Potential sources of latencies
  45. +
  46. +Potential sources of latencies are code segments where interrupts,
  47. +preemption or both are disabled (aka critical sections). To create
  48. +histograms of potential sources of latency, the kernel stores the time
  49. +stamp at the start of a critical section, determines the time elapsed
  50. +when the end of the section is reached, and increments the frequency
  51. +counter of that latency value - irrespective of whether any concurrently
  52. +running process is affected by latency or not.
  53. +- Configuration items (in the Kernel hacking/Tracers submenu)
  54. + CONFIG_INTERRUPT_OFF_LATENCY
  55. + CONFIG_PREEMPT_OFF_LATENCY
  56. +
  57. +
  58. +* Effective latencies
  59. +
  60. +Effective latencies are actually occuring during wakeup of a process. To
  61. +determine effective latencies, the kernel stores the time stamp when a
  62. +process is scheduled to be woken up, and determines the duration of the
  63. +wakeup time shortly before control is passed over to this process. Note
  64. +that the apparent latency in user space may be somewhat longer, since the
  65. +process may be interrupted after control is passed over to it but before
  66. +the execution in user space takes place. Simply measuring the interval
  67. +between enqueuing and wakeup may also not appropriate in cases when a
  68. +process is scheduled as a result of a timer expiration. The timer may have
  69. +missed its deadline, e.g. due to disabled interrupts, but this latency
  70. +would not be registered. Therefore, the offsets of missed timers are
  71. +recorded in a separate histogram. If both wakeup latency and missed timer
  72. +offsets are configured and enabled, a third histogram may be enabled that
  73. +records the overall latency as a sum of the timer latency, if any, and the
  74. +wakeup latency. This histogram is called "timerandwakeup".
  75. +- Configuration items (in the Kernel hacking/Tracers submenu)
  76. + CONFIG_WAKEUP_LATENCY
  77. + CONFIG_MISSED_TIMER_OFSETS
  78. +
  79. +
  80. +* Usage
  81. +
  82. +The interface to the administration of the latency histograms is located
  83. +in the debugfs file system. To mount it, either enter
  84. +
  85. +mount -t sysfs nodev /sys
  86. +mount -t debugfs nodev /sys/kernel/debug
  87. +
  88. +from shell command line level, or add
  89. +
  90. +nodev /sys sysfs defaults 0 0
  91. +nodev /sys/kernel/debug debugfs defaults 0 0
  92. +
  93. +to the file /etc/fstab. All latency histogram related files are then
  94. +available in the directory /sys/kernel/debug/tracing/latency_hist. A
  95. +particular histogram type is enabled by writing non-zero to the related
  96. +variable in the /sys/kernel/debug/tracing/latency_hist/enable directory.
  97. +Select "preemptirqsoff" for the histograms of potential sources of
  98. +latencies and "wakeup" for histograms of effective latencies etc. The
  99. +histogram data - one per CPU - are available in the files
  100. +
  101. +/sys/kernel/debug/tracing/latency_hist/preemptoff/CPUx
  102. +/sys/kernel/debug/tracing/latency_hist/irqsoff/CPUx
  103. +/sys/kernel/debug/tracing/latency_hist/preemptirqsoff/CPUx
  104. +/sys/kernel/debug/tracing/latency_hist/wakeup/CPUx
  105. +/sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio/CPUx
  106. +/sys/kernel/debug/tracing/latency_hist/missed_timer_offsets/CPUx
  107. +/sys/kernel/debug/tracing/latency_hist/timerandwakeup/CPUx
  108. +
  109. +The histograms are reset by writing non-zero to the file "reset" in a
  110. +particular latency directory. To reset all latency data, use
  111. +
  112. +#!/bin/sh
  113. +
  114. +TRACINGDIR=/sys/kernel/debug/tracing
  115. +HISTDIR=$TRACINGDIR/latency_hist
  116. +
  117. +if test -d $HISTDIR
  118. +then
  119. + cd $HISTDIR
  120. + for i in `find . | grep /reset$`
  121. + do
  122. + echo 1 >$i
  123. + done
  124. +fi
  125. +
  126. +
  127. +* Data format
  128. +
  129. +Latency data are stored with a resolution of one microsecond. The
  130. +maximum latency is 10,240 microseconds. The data are only valid, if the
  131. +overflow register is empty. Every output line contains the latency in
  132. +microseconds in the first row and the number of samples in the second
  133. +row. To display only lines with a positive latency count, use, for
  134. +example,
  135. +
  136. +grep -v " 0$" /sys/kernel/debug/tracing/latency_hist/preemptoff/CPU0
  137. +
  138. +#Minimum latency: 0 microseconds.
  139. +#Average latency: 0 microseconds.
  140. +#Maximum latency: 25 microseconds.
  141. +#Total samples: 3104770694
  142. +#There are 0 samples greater or equal than 10240 microseconds
  143. +#usecs samples
  144. + 0 2984486876
  145. + 1 49843506
  146. + 2 58219047
  147. + 3 5348126
  148. + 4 2187960
  149. + 5 3388262
  150. + 6 959289
  151. + 7 208294
  152. + 8 40420
  153. + 9 4485
  154. + 10 14918
  155. + 11 18340
  156. + 12 25052
  157. + 13 19455
  158. + 14 5602
  159. + 15 969
  160. + 16 47
  161. + 17 18
  162. + 18 14
  163. + 19 1
  164. + 20 3
  165. + 21 2
  166. + 22 5
  167. + 23 2
  168. + 25 1
  169. +
  170. +
  171. +* Wakeup latency of a selected process
  172. +
  173. +To only collect wakeup latency data of a particular process, write the
  174. +PID of the requested process to
  175. +
  176. +/sys/kernel/debug/tracing/latency_hist/wakeup/pid
  177. +
  178. +PIDs are not considered, if this variable is set to 0.
  179. +
  180. +
  181. +* Details of the process with the highest wakeup latency so far
  182. +
  183. +Selected data of the process that suffered from the highest wakeup
  184. +latency that occurred in a particular CPU are available in the file
  185. +
  186. +/sys/kernel/debug/tracing/latency_hist/wakeup/max_latency-CPUx.
  187. +
  188. +In addition, other relevant system data at the time when the
  189. +latency occurred are given.
  190. +
  191. +The format of the data is (all in one line):
  192. +<PID> <Priority> <Latency> (<Timeroffset>) <Command> \
  193. +<- <PID> <Priority> <Command> <Timestamp>
  194. +
  195. +The value of <Timeroffset> is only relevant in the combined timer
  196. +and wakeup latency recording. In the wakeup recording, it is
  197. +always 0, in the missed_timer_offsets recording, it is the same
  198. +as <Latency>.
  199. +
  200. +When retrospectively searching for the origin of a latency and
  201. +tracing was not enabled, it may be helpful to know the name and
  202. +some basic data of the task that (finally) was switching to the
  203. +late real-tlme task. In addition to the victim's data, also the
  204. +data of the possible culprit are therefore displayed after the
  205. +"<-" symbol.
  206. +
  207. +Finally, the timestamp of the time when the latency occurred
  208. +in <seconds>.<microseconds> after the most recent system boot
  209. +is provided.
  210. +
  211. +These data are also reset when the wakeup histogram is reset.
  212. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/Kconfig
  213. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/Kconfig 2017-04-16 10:37:28.000000000 +0200
  214. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/Kconfig 2017-04-18 17:54:20.000000000 +0200
  215. @@ -9,6 +9,7 @@
  216. tristate "OProfile system profiling"
  217. depends on PROFILING
  218. depends on HAVE_OPROFILE
  219. + depends on !PREEMPT_RT_FULL
  220. select RING_BUFFER
  221. select RING_BUFFER_ALLOW_SWAP
  222. help
  223. @@ -52,6 +53,7 @@
  224. config JUMP_LABEL
  225. bool "Optimize very unlikely/likely branches"
  226. depends on HAVE_ARCH_JUMP_LABEL
  227. + depends on (!INTERRUPT_OFF_HIST && !PREEMPT_OFF_HIST && !WAKEUP_LATENCY_HIST && !MISSED_TIMER_OFFSETS_HIST)
  228. help
  229. This option enables a transparent branch optimization that
  230. makes certain almost-always-true or almost-always-false branch
  231. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/Kconfig
  232. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/Kconfig 2017-04-16 10:37:29.000000000 +0200
  233. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/Kconfig 2017-04-18 17:54:19.000000000 +0200
  234. @@ -36,7 +36,7 @@
  235. select HAVE_ARCH_AUDITSYSCALL if (AEABI && !OABI_COMPAT)
  236. select HAVE_ARCH_BITREVERSE if (CPU_32v7M || CPU_32v7) && !CPU_32v6
  237. select HAVE_ARCH_HARDENED_USERCOPY
  238. - select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU
  239. + select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU && !PREEMPT_RT_BASE
  240. select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU
  241. select HAVE_ARCH_MMAP_RND_BITS if MMU
  242. select HAVE_ARCH_SECCOMP_FILTER if (AEABI && !OABI_COMPAT)
  243. @@ -75,6 +75,7 @@
  244. select HAVE_PERF_EVENTS
  245. select HAVE_PERF_REGS
  246. select HAVE_PERF_USER_STACK_DUMP
  247. + select HAVE_PREEMPT_LAZY
  248. select HAVE_RCU_TABLE_FREE if (SMP && ARM_LPAE)
  249. select HAVE_REGS_AND_STACK_ACCESS_API
  250. select HAVE_SYSCALL_TRACEPOINTS
  251. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/include/asm/irq.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/include/asm/irq.h
  252. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/include/asm/irq.h 2017-04-16 10:37:30.000000000 +0200
  253. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/include/asm/irq.h 2017-04-18 17:54:19.000000000 +0200
  254. @@ -22,6 +22,8 @@
  255. #endif
  256. #ifndef __ASSEMBLY__
  257. +#include <linux/cpumask.h>
  258. +
  259. struct irqaction;
  260. struct pt_regs;
  261. extern void migrate_irqs(void);
  262. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/include/asm/switch_to.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/include/asm/switch_to.h
  263. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/include/asm/switch_to.h 2017-04-16 10:37:30.000000000 +0200
  264. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/include/asm/switch_to.h 2017-04-18 17:54:19.000000000 +0200
  265. @@ -3,6 +3,13 @@
  266. #include <linux/thread_info.h>
  267. +#if defined CONFIG_PREEMPT_RT_FULL && defined CONFIG_HIGHMEM
  268. +void switch_kmaps(struct task_struct *prev_p, struct task_struct *next_p);
  269. +#else
  270. +static inline void
  271. +switch_kmaps(struct task_struct *prev_p, struct task_struct *next_p) { }
  272. +#endif
  273. +
  274. /*
  275. * For v7 SMP cores running a preemptible kernel we may be pre-empted
  276. * during a TLB maintenance operation, so execute an inner-shareable dsb
  277. @@ -25,6 +32,7 @@
  278. #define switch_to(prev,next,last) \
  279. do { \
  280. __complete_pending_tlbi(); \
  281. + switch_kmaps(prev, next); \
  282. last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \
  283. } while (0)
  284. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/include/asm/thread_info.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/include/asm/thread_info.h
  285. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/include/asm/thread_info.h 2017-04-16 10:37:30.000000000 +0200
  286. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/include/asm/thread_info.h 2017-04-18 17:54:19.000000000 +0200
  287. @@ -49,6 +49,7 @@
  288. struct thread_info {
  289. unsigned long flags; /* low level flags */
  290. int preempt_count; /* 0 => preemptable, <0 => bug */
  291. + int preempt_lazy_count; /* 0 => preemptable, <0 => bug */
  292. mm_segment_t addr_limit; /* address limit */
  293. struct task_struct *task; /* main task structure */
  294. __u32 cpu; /* cpu */
  295. @@ -142,7 +143,8 @@
  296. #define TIF_SYSCALL_TRACE 4 /* syscall trace active */
  297. #define TIF_SYSCALL_AUDIT 5 /* syscall auditing active */
  298. #define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */
  299. -#define TIF_SECCOMP 7 /* seccomp syscall filtering active */
  300. +#define TIF_SECCOMP 8 /* seccomp syscall filtering active */
  301. +#define TIF_NEED_RESCHED_LAZY 7
  302. #define TIF_NOHZ 12 /* in adaptive nohz mode */
  303. #define TIF_USING_IWMMXT 17
  304. @@ -152,6 +154,7 @@
  305. #define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
  306. #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
  307. #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
  308. +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY)
  309. #define _TIF_UPROBE (1 << TIF_UPROBE)
  310. #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
  311. #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
  312. @@ -167,7 +170,8 @@
  313. * Change these and you break ASM code in entry-common.S
  314. */
  315. #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
  316. - _TIF_NOTIFY_RESUME | _TIF_UPROBE)
  317. + _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
  318. + _TIF_NEED_RESCHED_LAZY)
  319. #endif /* __KERNEL__ */
  320. #endif /* __ASM_ARM_THREAD_INFO_H */
  321. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/asm-offsets.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/asm-offsets.c
  322. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/asm-offsets.c 2017-04-16 10:37:30.000000000 +0200
  323. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/asm-offsets.c 2017-04-18 17:54:19.000000000 +0200
  324. @@ -65,6 +65,7 @@
  325. BLANK();
  326. DEFINE(TI_FLAGS, offsetof(struct thread_info, flags));
  327. DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count));
  328. + DEFINE(TI_PREEMPT_LAZY, offsetof(struct thread_info, preempt_lazy_count));
  329. DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit));
  330. DEFINE(TI_TASK, offsetof(struct thread_info, task));
  331. DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
  332. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/entry-armv.S linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/entry-armv.S
  333. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/entry-armv.S 2017-04-16 10:37:30.000000000 +0200
  334. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/entry-armv.S 2017-04-18 17:54:19.000000000 +0200
  335. @@ -220,11 +220,18 @@
  336. #ifdef CONFIG_PREEMPT
  337. ldr r8, [tsk, #TI_PREEMPT] @ get preempt count
  338. - ldr r0, [tsk, #TI_FLAGS] @ get flags
  339. teq r8, #0 @ if preempt count != 0
  340. + bne 1f @ return from exeption
  341. + ldr r0, [tsk, #TI_FLAGS] @ get flags
  342. + tst r0, #_TIF_NEED_RESCHED @ if NEED_RESCHED is set
  343. + blne svc_preempt @ preempt!
  344. +
  345. + ldr r8, [tsk, #TI_PREEMPT_LAZY] @ get preempt lazy count
  346. + teq r8, #0 @ if preempt lazy count != 0
  347. movne r0, #0 @ force flags to 0
  348. - tst r0, #_TIF_NEED_RESCHED
  349. + tst r0, #_TIF_NEED_RESCHED_LAZY
  350. blne svc_preempt
  351. +1:
  352. #endif
  353. svc_exit r5, irq = 1 @ return from exception
  354. @@ -239,8 +246,14 @@
  355. 1: bl preempt_schedule_irq @ irq en/disable is done inside
  356. ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS
  357. tst r0, #_TIF_NEED_RESCHED
  358. + bne 1b
  359. + tst r0, #_TIF_NEED_RESCHED_LAZY
  360. reteq r8 @ go again
  361. - b 1b
  362. + ldr r0, [tsk, #TI_PREEMPT_LAZY] @ get preempt lazy count
  363. + teq r0, #0 @ if preempt lazy count != 0
  364. + beq 1b
  365. + ret r8 @ go again
  366. +
  367. #endif
  368. __und_fault:
  369. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/entry-common.S linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/entry-common.S
  370. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/entry-common.S 2017-04-16 10:37:30.000000000 +0200
  371. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/entry-common.S 2017-04-18 17:54:19.000000000 +0200
  372. @@ -36,7 +36,9 @@
  373. UNWIND(.cantunwind )
  374. disable_irq_notrace @ disable interrupts
  375. ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing
  376. - tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
  377. + tst r1, #((_TIF_SYSCALL_WORK | _TIF_WORK_MASK) & ~_TIF_SECCOMP)
  378. + bne fast_work_pending
  379. + tst r1, #_TIF_SECCOMP
  380. bne fast_work_pending
  381. /* perform architecture specific actions before user return */
  382. @@ -62,8 +64,11 @@
  383. str r0, [sp, #S_R0 + S_OFF]! @ save returned r0
  384. disable_irq_notrace @ disable interrupts
  385. ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing
  386. - tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
  387. + tst r1, #((_TIF_SYSCALL_WORK | _TIF_WORK_MASK) & ~_TIF_SECCOMP)
  388. + bne do_slower_path
  389. + tst r1, #_TIF_SECCOMP
  390. beq no_work_pending
  391. +do_slower_path:
  392. UNWIND(.fnend )
  393. ENDPROC(ret_fast_syscall)
  394. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/patch.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/patch.c
  395. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/patch.c 2017-04-16 10:37:30.000000000 +0200
  396. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/patch.c 2017-04-18 17:54:19.000000000 +0200
  397. @@ -15,7 +15,7 @@
  398. unsigned int insn;
  399. };
  400. -static DEFINE_SPINLOCK(patch_lock);
  401. +static DEFINE_RAW_SPINLOCK(patch_lock);
  402. static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags)
  403. __acquires(&patch_lock)
  404. @@ -32,7 +32,7 @@
  405. return addr;
  406. if (flags)
  407. - spin_lock_irqsave(&patch_lock, *flags);
  408. + raw_spin_lock_irqsave(&patch_lock, *flags);
  409. else
  410. __acquire(&patch_lock);
  411. @@ -47,7 +47,7 @@
  412. clear_fixmap(fixmap);
  413. if (flags)
  414. - spin_unlock_irqrestore(&patch_lock, *flags);
  415. + raw_spin_unlock_irqrestore(&patch_lock, *flags);
  416. else
  417. __release(&patch_lock);
  418. }
  419. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/process.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/process.c
  420. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/process.c 2017-04-16 10:37:30.000000000 +0200
  421. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/process.c 2017-04-18 17:54:19.000000000 +0200
  422. @@ -322,6 +322,30 @@
  423. }
  424. #ifdef CONFIG_MMU
  425. +/*
  426. + * CONFIG_SPLIT_PTLOCK_CPUS results in a page->ptl lock. If the lock is not
  427. + * initialized by pgtable_page_ctor() then a coredump of the vector page will
  428. + * fail.
  429. + */
  430. +static int __init vectors_user_mapping_init_page(void)
  431. +{
  432. + struct page *page;
  433. + unsigned long addr = 0xffff0000;
  434. + pgd_t *pgd;
  435. + pud_t *pud;
  436. + pmd_t *pmd;
  437. +
  438. + pgd = pgd_offset_k(addr);
  439. + pud = pud_offset(pgd, addr);
  440. + pmd = pmd_offset(pud, addr);
  441. + page = pmd_page(*(pmd));
  442. +
  443. + pgtable_page_ctor(page);
  444. +
  445. + return 0;
  446. +}
  447. +late_initcall(vectors_user_mapping_init_page);
  448. +
  449. #ifdef CONFIG_KUSER_HELPERS
  450. /*
  451. * The vectors page is always readable from user space for the
  452. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/signal.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/signal.c
  453. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/signal.c 2017-04-16 10:37:30.000000000 +0200
  454. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/signal.c 2017-04-18 17:54:19.000000000 +0200
  455. @@ -572,7 +572,8 @@
  456. */
  457. trace_hardirqs_off();
  458. do {
  459. - if (likely(thread_flags & _TIF_NEED_RESCHED)) {
  460. + if (likely(thread_flags & (_TIF_NEED_RESCHED |
  461. + _TIF_NEED_RESCHED_LAZY))) {
  462. schedule();
  463. } else {
  464. if (unlikely(!user_mode(regs)))
  465. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/smp.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/smp.c
  466. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/smp.c 2017-04-16 10:37:30.000000000 +0200
  467. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/smp.c 2017-04-18 17:54:19.000000000 +0200
  468. @@ -234,8 +234,6 @@
  469. flush_cache_louis();
  470. local_flush_tlb_all();
  471. - clear_tasks_mm_cpumask(cpu);
  472. -
  473. return 0;
  474. }
  475. @@ -251,6 +249,9 @@
  476. pr_err("CPU%u: cpu didn't die\n", cpu);
  477. return;
  478. }
  479. +
  480. + clear_tasks_mm_cpumask(cpu);
  481. +
  482. pr_notice("CPU%u: shutdown\n", cpu);
  483. /*
  484. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/unwind.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/unwind.c
  485. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kernel/unwind.c 2017-04-16 10:37:30.000000000 +0200
  486. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kernel/unwind.c 2017-04-18 17:54:19.000000000 +0200
  487. @@ -93,7 +93,7 @@
  488. static const struct unwind_idx *__origin_unwind_idx;
  489. extern const struct unwind_idx __stop_unwind_idx[];
  490. -static DEFINE_SPINLOCK(unwind_lock);
  491. +static DEFINE_RAW_SPINLOCK(unwind_lock);
  492. static LIST_HEAD(unwind_tables);
  493. /* Convert a prel31 symbol to an absolute address */
  494. @@ -201,7 +201,7 @@
  495. /* module unwind tables */
  496. struct unwind_table *table;
  497. - spin_lock_irqsave(&unwind_lock, flags);
  498. + raw_spin_lock_irqsave(&unwind_lock, flags);
  499. list_for_each_entry(table, &unwind_tables, list) {
  500. if (addr >= table->begin_addr &&
  501. addr < table->end_addr) {
  502. @@ -213,7 +213,7 @@
  503. break;
  504. }
  505. }
  506. - spin_unlock_irqrestore(&unwind_lock, flags);
  507. + raw_spin_unlock_irqrestore(&unwind_lock, flags);
  508. }
  509. pr_debug("%s: idx = %p\n", __func__, idx);
  510. @@ -529,9 +529,9 @@
  511. tab->begin_addr = text_addr;
  512. tab->end_addr = text_addr + text_size;
  513. - spin_lock_irqsave(&unwind_lock, flags);
  514. + raw_spin_lock_irqsave(&unwind_lock, flags);
  515. list_add_tail(&tab->list, &unwind_tables);
  516. - spin_unlock_irqrestore(&unwind_lock, flags);
  517. + raw_spin_unlock_irqrestore(&unwind_lock, flags);
  518. return tab;
  519. }
  520. @@ -543,9 +543,9 @@
  521. if (!tab)
  522. return;
  523. - spin_lock_irqsave(&unwind_lock, flags);
  524. + raw_spin_lock_irqsave(&unwind_lock, flags);
  525. list_del(&tab->list);
  526. - spin_unlock_irqrestore(&unwind_lock, flags);
  527. + raw_spin_unlock_irqrestore(&unwind_lock, flags);
  528. kfree(tab);
  529. }
  530. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kvm/arm.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kvm/arm.c
  531. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/kvm/arm.c 2017-04-16 10:37:30.000000000 +0200
  532. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/kvm/arm.c 2017-04-18 17:54:19.000000000 +0200
  533. @@ -619,7 +619,7 @@
  534. * involves poking the GIC, which must be done in a
  535. * non-preemptible context.
  536. */
  537. - preempt_disable();
  538. + migrate_disable();
  539. kvm_pmu_flush_hwstate(vcpu);
  540. kvm_timer_flush_hwstate(vcpu);
  541. kvm_vgic_flush_hwstate(vcpu);
  542. @@ -640,7 +640,7 @@
  543. kvm_pmu_sync_hwstate(vcpu);
  544. kvm_timer_sync_hwstate(vcpu);
  545. kvm_vgic_sync_hwstate(vcpu);
  546. - preempt_enable();
  547. + migrate_enable();
  548. continue;
  549. }
  550. @@ -696,7 +696,7 @@
  551. kvm_vgic_sync_hwstate(vcpu);
  552. - preempt_enable();
  553. + migrate_enable();
  554. ret = handle_exit(vcpu, run, ret);
  555. }
  556. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-exynos/platsmp.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-exynos/platsmp.c
  557. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-exynos/platsmp.c 2017-04-16 10:37:30.000000000 +0200
  558. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-exynos/platsmp.c 2017-04-18 17:54:19.000000000 +0200
  559. @@ -229,7 +229,7 @@
  560. return (void __iomem *)(S5P_VA_SCU);
  561. }
  562. -static DEFINE_SPINLOCK(boot_lock);
  563. +static DEFINE_RAW_SPINLOCK(boot_lock);
  564. static void exynos_secondary_init(unsigned int cpu)
  565. {
  566. @@ -242,8 +242,8 @@
  567. /*
  568. * Synchronise with the boot thread.
  569. */
  570. - spin_lock(&boot_lock);
  571. - spin_unlock(&boot_lock);
  572. + raw_spin_lock(&boot_lock);
  573. + raw_spin_unlock(&boot_lock);
  574. }
  575. int exynos_set_boot_addr(u32 core_id, unsigned long boot_addr)
  576. @@ -307,7 +307,7 @@
  577. * Set synchronisation state between this boot processor
  578. * and the secondary one
  579. */
  580. - spin_lock(&boot_lock);
  581. + raw_spin_lock(&boot_lock);
  582. /*
  583. * The secondary processor is waiting to be released from
  584. @@ -334,7 +334,7 @@
  585. if (timeout == 0) {
  586. printk(KERN_ERR "cpu1 power enable failed");
  587. - spin_unlock(&boot_lock);
  588. + raw_spin_unlock(&boot_lock);
  589. return -ETIMEDOUT;
  590. }
  591. }
  592. @@ -380,7 +380,7 @@
  593. * calibrations, then wait for it to finish
  594. */
  595. fail:
  596. - spin_unlock(&boot_lock);
  597. + raw_spin_unlock(&boot_lock);
  598. return pen_release != -1 ? ret : 0;
  599. }
  600. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-hisi/platmcpm.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-hisi/platmcpm.c
  601. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-hisi/platmcpm.c 2017-04-16 10:37:30.000000000 +0200
  602. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-hisi/platmcpm.c 2017-04-18 17:54:19.000000000 +0200
  603. @@ -61,7 +61,7 @@
  604. static void __iomem *sysctrl, *fabric;
  605. static int hip04_cpu_table[HIP04_MAX_CLUSTERS][HIP04_MAX_CPUS_PER_CLUSTER];
  606. -static DEFINE_SPINLOCK(boot_lock);
  607. +static DEFINE_RAW_SPINLOCK(boot_lock);
  608. static u32 fabric_phys_addr;
  609. /*
  610. * [0]: bootwrapper physical address
  611. @@ -113,7 +113,7 @@
  612. if (cluster >= HIP04_MAX_CLUSTERS || cpu >= HIP04_MAX_CPUS_PER_CLUSTER)
  613. return -EINVAL;
  614. - spin_lock_irq(&boot_lock);
  615. + raw_spin_lock_irq(&boot_lock);
  616. if (hip04_cpu_table[cluster][cpu])
  617. goto out;
  618. @@ -147,7 +147,7 @@
  619. out:
  620. hip04_cpu_table[cluster][cpu]++;
  621. - spin_unlock_irq(&boot_lock);
  622. + raw_spin_unlock_irq(&boot_lock);
  623. return 0;
  624. }
  625. @@ -162,11 +162,11 @@
  626. cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
  627. cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
  628. - spin_lock(&boot_lock);
  629. + raw_spin_lock(&boot_lock);
  630. hip04_cpu_table[cluster][cpu]--;
  631. if (hip04_cpu_table[cluster][cpu] == 1) {
  632. /* A power_up request went ahead of us. */
  633. - spin_unlock(&boot_lock);
  634. + raw_spin_unlock(&boot_lock);
  635. return;
  636. } else if (hip04_cpu_table[cluster][cpu] > 1) {
  637. pr_err("Cluster %d CPU%d boots multiple times\n", cluster, cpu);
  638. @@ -174,7 +174,7 @@
  639. }
  640. last_man = hip04_cluster_is_down(cluster);
  641. - spin_unlock(&boot_lock);
  642. + raw_spin_unlock(&boot_lock);
  643. if (last_man) {
  644. /* Since it's Cortex A15, disable L2 prefetching. */
  645. asm volatile(
  646. @@ -203,7 +203,7 @@
  647. cpu >= HIP04_MAX_CPUS_PER_CLUSTER);
  648. count = TIMEOUT_MSEC / POLL_MSEC;
  649. - spin_lock_irq(&boot_lock);
  650. + raw_spin_lock_irq(&boot_lock);
  651. for (tries = 0; tries < count; tries++) {
  652. if (hip04_cpu_table[cluster][cpu])
  653. goto err;
  654. @@ -211,10 +211,10 @@
  655. data = readl_relaxed(sysctrl + SC_CPU_RESET_STATUS(cluster));
  656. if (data & CORE_WFI_STATUS(cpu))
  657. break;
  658. - spin_unlock_irq(&boot_lock);
  659. + raw_spin_unlock_irq(&boot_lock);
  660. /* Wait for clean L2 when the whole cluster is down. */
  661. msleep(POLL_MSEC);
  662. - spin_lock_irq(&boot_lock);
  663. + raw_spin_lock_irq(&boot_lock);
  664. }
  665. if (tries >= count)
  666. goto err;
  667. @@ -231,10 +231,10 @@
  668. goto err;
  669. if (hip04_cluster_is_down(cluster))
  670. hip04_set_snoop_filter(cluster, 0);
  671. - spin_unlock_irq(&boot_lock);
  672. + raw_spin_unlock_irq(&boot_lock);
  673. return 1;
  674. err:
  675. - spin_unlock_irq(&boot_lock);
  676. + raw_spin_unlock_irq(&boot_lock);
  677. return 0;
  678. }
  679. #endif
  680. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-omap2/omap-smp.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-omap2/omap-smp.c
  681. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-omap2/omap-smp.c 2017-04-16 10:37:31.000000000 +0200
  682. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-omap2/omap-smp.c 2017-04-18 17:54:19.000000000 +0200
  683. @@ -64,7 +64,7 @@
  684. .startup_addr = omap5_secondary_startup,
  685. };
  686. -static DEFINE_SPINLOCK(boot_lock);
  687. +static DEFINE_RAW_SPINLOCK(boot_lock);
  688. void __iomem *omap4_get_scu_base(void)
  689. {
  690. @@ -131,8 +131,8 @@
  691. /*
  692. * Synchronise with the boot thread.
  693. */
  694. - spin_lock(&boot_lock);
  695. - spin_unlock(&boot_lock);
  696. + raw_spin_lock(&boot_lock);
  697. + raw_spin_unlock(&boot_lock);
  698. }
  699. static int omap4_boot_secondary(unsigned int cpu, struct task_struct *idle)
  700. @@ -146,7 +146,7 @@
  701. * Set synchronisation state between this boot processor
  702. * and the secondary one
  703. */
  704. - spin_lock(&boot_lock);
  705. + raw_spin_lock(&boot_lock);
  706. /*
  707. * Update the AuxCoreBoot0 with boot state for secondary core.
  708. @@ -223,7 +223,7 @@
  709. * Now the secondary core is starting up let it run its
  710. * calibrations, then wait for it to finish
  711. */
  712. - spin_unlock(&boot_lock);
  713. + raw_spin_unlock(&boot_lock);
  714. return 0;
  715. }
  716. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-prima2/platsmp.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-prima2/platsmp.c
  717. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-prima2/platsmp.c 2017-04-16 10:37:31.000000000 +0200
  718. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-prima2/platsmp.c 2017-04-18 17:54:19.000000000 +0200
  719. @@ -22,7 +22,7 @@
  720. static void __iomem *clk_base;
  721. -static DEFINE_SPINLOCK(boot_lock);
  722. +static DEFINE_RAW_SPINLOCK(boot_lock);
  723. static void sirfsoc_secondary_init(unsigned int cpu)
  724. {
  725. @@ -36,8 +36,8 @@
  726. /*
  727. * Synchronise with the boot thread.
  728. */
  729. - spin_lock(&boot_lock);
  730. - spin_unlock(&boot_lock);
  731. + raw_spin_lock(&boot_lock);
  732. + raw_spin_unlock(&boot_lock);
  733. }
  734. static const struct of_device_id clk_ids[] = {
  735. @@ -75,7 +75,7 @@
  736. /* make sure write buffer is drained */
  737. mb();
  738. - spin_lock(&boot_lock);
  739. + raw_spin_lock(&boot_lock);
  740. /*
  741. * The secondary processor is waiting to be released from
  742. @@ -107,7 +107,7 @@
  743. * now the secondary core is starting up let it run its
  744. * calibrations, then wait for it to finish
  745. */
  746. - spin_unlock(&boot_lock);
  747. + raw_spin_unlock(&boot_lock);
  748. return pen_release != -1 ? -ENOSYS : 0;
  749. }
  750. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-qcom/platsmp.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-qcom/platsmp.c
  751. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-qcom/platsmp.c 2017-04-16 10:37:31.000000000 +0200
  752. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-qcom/platsmp.c 2017-04-18 17:54:20.000000000 +0200
  753. @@ -46,7 +46,7 @@
  754. extern void secondary_startup_arm(void);
  755. -static DEFINE_SPINLOCK(boot_lock);
  756. +static DEFINE_RAW_SPINLOCK(boot_lock);
  757. #ifdef CONFIG_HOTPLUG_CPU
  758. static void qcom_cpu_die(unsigned int cpu)
  759. @@ -60,8 +60,8 @@
  760. /*
  761. * Synchronise with the boot thread.
  762. */
  763. - spin_lock(&boot_lock);
  764. - spin_unlock(&boot_lock);
  765. + raw_spin_lock(&boot_lock);
  766. + raw_spin_unlock(&boot_lock);
  767. }
  768. static int scss_release_secondary(unsigned int cpu)
  769. @@ -284,7 +284,7 @@
  770. * set synchronisation state between this boot processor
  771. * and the secondary one
  772. */
  773. - spin_lock(&boot_lock);
  774. + raw_spin_lock(&boot_lock);
  775. /*
  776. * Send the secondary CPU a soft interrupt, thereby causing
  777. @@ -297,7 +297,7 @@
  778. * now the secondary core is starting up let it run its
  779. * calibrations, then wait for it to finish
  780. */
  781. - spin_unlock(&boot_lock);
  782. + raw_spin_unlock(&boot_lock);
  783. return ret;
  784. }
  785. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-spear/platsmp.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-spear/platsmp.c
  786. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-spear/platsmp.c 2017-04-16 10:37:31.000000000 +0200
  787. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-spear/platsmp.c 2017-04-18 17:54:20.000000000 +0200
  788. @@ -32,7 +32,7 @@
  789. sync_cache_w(&pen_release);
  790. }
  791. -static DEFINE_SPINLOCK(boot_lock);
  792. +static DEFINE_RAW_SPINLOCK(boot_lock);
  793. static void __iomem *scu_base = IOMEM(VA_SCU_BASE);
  794. @@ -47,8 +47,8 @@
  795. /*
  796. * Synchronise with the boot thread.
  797. */
  798. - spin_lock(&boot_lock);
  799. - spin_unlock(&boot_lock);
  800. + raw_spin_lock(&boot_lock);
  801. + raw_spin_unlock(&boot_lock);
  802. }
  803. static int spear13xx_boot_secondary(unsigned int cpu, struct task_struct *idle)
  804. @@ -59,7 +59,7 @@
  805. * set synchronisation state between this boot processor
  806. * and the secondary one
  807. */
  808. - spin_lock(&boot_lock);
  809. + raw_spin_lock(&boot_lock);
  810. /*
  811. * The secondary processor is waiting to be released from
  812. @@ -84,7 +84,7 @@
  813. * now the secondary core is starting up let it run its
  814. * calibrations, then wait for it to finish
  815. */
  816. - spin_unlock(&boot_lock);
  817. + raw_spin_unlock(&boot_lock);
  818. return pen_release != -1 ? -ENOSYS : 0;
  819. }
  820. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-sti/platsmp.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-sti/platsmp.c
  821. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mach-sti/platsmp.c 2017-04-16 10:37:31.000000000 +0200
  822. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mach-sti/platsmp.c 2017-04-18 17:54:20.000000000 +0200
  823. @@ -35,7 +35,7 @@
  824. sync_cache_w(&pen_release);
  825. }
  826. -static DEFINE_SPINLOCK(boot_lock);
  827. +static DEFINE_RAW_SPINLOCK(boot_lock);
  828. static void sti_secondary_init(unsigned int cpu)
  829. {
  830. @@ -48,8 +48,8 @@
  831. /*
  832. * Synchronise with the boot thread.
  833. */
  834. - spin_lock(&boot_lock);
  835. - spin_unlock(&boot_lock);
  836. + raw_spin_lock(&boot_lock);
  837. + raw_spin_unlock(&boot_lock);
  838. }
  839. static int sti_boot_secondary(unsigned int cpu, struct task_struct *idle)
  840. @@ -60,7 +60,7 @@
  841. * set synchronisation state between this boot processor
  842. * and the secondary one
  843. */
  844. - spin_lock(&boot_lock);
  845. + raw_spin_lock(&boot_lock);
  846. /*
  847. * The secondary processor is waiting to be released from
  848. @@ -91,7 +91,7 @@
  849. * now the secondary core is starting up let it run its
  850. * calibrations, then wait for it to finish
  851. */
  852. - spin_unlock(&boot_lock);
  853. + raw_spin_unlock(&boot_lock);
  854. return pen_release != -1 ? -ENOSYS : 0;
  855. }
  856. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mm/fault.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mm/fault.c
  857. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mm/fault.c 2017-04-16 10:37:31.000000000 +0200
  858. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mm/fault.c 2017-04-18 17:54:20.000000000 +0200
  859. @@ -430,6 +430,9 @@
  860. if (addr < TASK_SIZE)
  861. return do_page_fault(addr, fsr, regs);
  862. + if (interrupts_enabled(regs))
  863. + local_irq_enable();
  864. +
  865. if (user_mode(regs))
  866. goto bad_area;
  867. @@ -497,6 +500,9 @@
  868. static int
  869. do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
  870. {
  871. + if (interrupts_enabled(regs))
  872. + local_irq_enable();
  873. +
  874. do_bad_area(addr, fsr, regs);
  875. return 0;
  876. }
  877. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mm/highmem.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mm/highmem.c
  878. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/mm/highmem.c 2017-04-16 10:37:31.000000000 +0200
  879. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/mm/highmem.c 2017-04-18 17:54:20.000000000 +0200
  880. @@ -34,6 +34,11 @@
  881. return *ptep;
  882. }
  883. +static unsigned int fixmap_idx(int type)
  884. +{
  885. + return FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id();
  886. +}
  887. +
  888. void *kmap(struct page *page)
  889. {
  890. might_sleep();
  891. @@ -54,12 +59,13 @@
  892. void *kmap_atomic(struct page *page)
  893. {
  894. + pte_t pte = mk_pte(page, kmap_prot);
  895. unsigned int idx;
  896. unsigned long vaddr;
  897. void *kmap;
  898. int type;
  899. - preempt_disable();
  900. + preempt_disable_nort();
  901. pagefault_disable();
  902. if (!PageHighMem(page))
  903. return page_address(page);
  904. @@ -79,7 +85,7 @@
  905. type = kmap_atomic_idx_push();
  906. - idx = FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id();
  907. + idx = fixmap_idx(type);
  908. vaddr = __fix_to_virt(idx);
  909. #ifdef CONFIG_DEBUG_HIGHMEM
  910. /*
  911. @@ -93,7 +99,10 @@
  912. * in place, so the contained TLB flush ensures the TLB is updated
  913. * with the new mapping.
  914. */
  915. - set_fixmap_pte(idx, mk_pte(page, kmap_prot));
  916. +#ifdef CONFIG_PREEMPT_RT_FULL
  917. + current->kmap_pte[type] = pte;
  918. +#endif
  919. + set_fixmap_pte(idx, pte);
  920. return (void *)vaddr;
  921. }
  922. @@ -106,44 +115,75 @@
  923. if (kvaddr >= (void *)FIXADDR_START) {
  924. type = kmap_atomic_idx();
  925. - idx = FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id();
  926. + idx = fixmap_idx(type);
  927. if (cache_is_vivt())
  928. __cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE);
  929. +#ifdef CONFIG_PREEMPT_RT_FULL
  930. + current->kmap_pte[type] = __pte(0);
  931. +#endif
  932. #ifdef CONFIG_DEBUG_HIGHMEM
  933. BUG_ON(vaddr != __fix_to_virt(idx));
  934. - set_fixmap_pte(idx, __pte(0));
  935. #else
  936. (void) idx; /* to kill a warning */
  937. #endif
  938. + set_fixmap_pte(idx, __pte(0));
  939. kmap_atomic_idx_pop();
  940. } else if (vaddr >= PKMAP_ADDR(0) && vaddr < PKMAP_ADDR(LAST_PKMAP)) {
  941. /* this address was obtained through kmap_high_get() */
  942. kunmap_high(pte_page(pkmap_page_table[PKMAP_NR(vaddr)]));
  943. }
  944. pagefault_enable();
  945. - preempt_enable();
  946. + preempt_enable_nort();
  947. }
  948. EXPORT_SYMBOL(__kunmap_atomic);
  949. void *kmap_atomic_pfn(unsigned long pfn)
  950. {
  951. + pte_t pte = pfn_pte(pfn, kmap_prot);
  952. unsigned long vaddr;
  953. int idx, type;
  954. struct page *page = pfn_to_page(pfn);
  955. - preempt_disable();
  956. + preempt_disable_nort();
  957. pagefault_disable();
  958. if (!PageHighMem(page))
  959. return page_address(page);
  960. type = kmap_atomic_idx_push();
  961. - idx = FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id();
  962. + idx = fixmap_idx(type);
  963. vaddr = __fix_to_virt(idx);
  964. #ifdef CONFIG_DEBUG_HIGHMEM
  965. BUG_ON(!pte_none(get_fixmap_pte(vaddr)));
  966. #endif
  967. - set_fixmap_pte(idx, pfn_pte(pfn, kmap_prot));
  968. +#ifdef CONFIG_PREEMPT_RT_FULL
  969. + current->kmap_pte[type] = pte;
  970. +#endif
  971. + set_fixmap_pte(idx, pte);
  972. return (void *)vaddr;
  973. }
  974. +#if defined CONFIG_PREEMPT_RT_FULL
  975. +void switch_kmaps(struct task_struct *prev_p, struct task_struct *next_p)
  976. +{
  977. + int i;
  978. +
  979. + /*
  980. + * Clear @prev's kmap_atomic mappings
  981. + */
  982. + for (i = 0; i < prev_p->kmap_idx; i++) {
  983. + int idx = fixmap_idx(i);
  984. +
  985. + set_fixmap_pte(idx, __pte(0));
  986. + }
  987. + /*
  988. + * Restore @next_p's kmap_atomic mappings
  989. + */
  990. + for (i = 0; i < next_p->kmap_idx; i++) {
  991. + int idx = fixmap_idx(i);
  992. +
  993. + if (!pte_none(next_p->kmap_pte[i]))
  994. + set_fixmap_pte(idx, next_p->kmap_pte[i]);
  995. + }
  996. +}
  997. +#endif
  998. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/plat-versatile/platsmp.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/plat-versatile/platsmp.c
  999. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm/plat-versatile/platsmp.c 2017-04-16 10:37:31.000000000 +0200
  1000. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm/plat-versatile/platsmp.c 2017-04-18 17:54:20.000000000 +0200
  1001. @@ -32,7 +32,7 @@
  1002. sync_cache_w(&pen_release);
  1003. }
  1004. -static DEFINE_SPINLOCK(boot_lock);
  1005. +static DEFINE_RAW_SPINLOCK(boot_lock);
  1006. void versatile_secondary_init(unsigned int cpu)
  1007. {
  1008. @@ -45,8 +45,8 @@
  1009. /*
  1010. * Synchronise with the boot thread.
  1011. */
  1012. - spin_lock(&boot_lock);
  1013. - spin_unlock(&boot_lock);
  1014. + raw_spin_lock(&boot_lock);
  1015. + raw_spin_unlock(&boot_lock);
  1016. }
  1017. int versatile_boot_secondary(unsigned int cpu, struct task_struct *idle)
  1018. @@ -57,7 +57,7 @@
  1019. * Set synchronisation state between this boot processor
  1020. * and the secondary one
  1021. */
  1022. - spin_lock(&boot_lock);
  1023. + raw_spin_lock(&boot_lock);
  1024. /*
  1025. * This is really belt and braces; we hold unintended secondary
  1026. @@ -87,7 +87,7 @@
  1027. * now the secondary core is starting up let it run its
  1028. * calibrations, then wait for it to finish
  1029. */
  1030. - spin_unlock(&boot_lock);
  1031. + raw_spin_unlock(&boot_lock);
  1032. return pen_release != -1 ? -ENOSYS : 0;
  1033. }
  1034. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm64/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm64/Kconfig
  1035. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm64/Kconfig 2017-04-16 10:37:31.000000000 +0200
  1036. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm64/Kconfig 2017-04-18 17:54:20.000000000 +0200
  1037. @@ -91,6 +91,7 @@
  1038. select HAVE_PERF_EVENTS
  1039. select HAVE_PERF_REGS
  1040. select HAVE_PERF_USER_STACK_DUMP
  1041. + select HAVE_PREEMPT_LAZY
  1042. select HAVE_REGS_AND_STACK_ACCESS_API
  1043. select HAVE_RCU_TABLE_FREE
  1044. select HAVE_SYSCALL_TRACEPOINTS
  1045. @@ -704,7 +705,7 @@
  1046. config XEN
  1047. bool "Xen guest support on ARM64"
  1048. - depends on ARM64 && OF
  1049. + depends on ARM64 && OF && !PREEMPT_RT_FULL
  1050. select SWIOTLB_XEN
  1051. select PARAVIRT
  1052. help
  1053. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm64/include/asm/thread_info.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm64/include/asm/thread_info.h
  1054. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm64/include/asm/thread_info.h 2017-04-16 10:37:32.000000000 +0200
  1055. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm64/include/asm/thread_info.h 2017-04-18 17:54:20.000000000 +0200
  1056. @@ -49,6 +49,7 @@
  1057. mm_segment_t addr_limit; /* address limit */
  1058. struct task_struct *task; /* main task structure */
  1059. int preempt_count; /* 0 => preemptable, <0 => bug */
  1060. + int preempt_lazy_count; /* 0 => preemptable, <0 => bug */
  1061. int cpu; /* cpu */
  1062. };
  1063. @@ -112,6 +113,7 @@
  1064. #define TIF_NEED_RESCHED 1
  1065. #define TIF_NOTIFY_RESUME 2 /* callback before returning to user */
  1066. #define TIF_FOREIGN_FPSTATE 3 /* CPU's FP state is not current's */
  1067. +#define TIF_NEED_RESCHED_LAZY 4
  1068. #define TIF_NOHZ 7
  1069. #define TIF_SYSCALL_TRACE 8
  1070. #define TIF_SYSCALL_AUDIT 9
  1071. @@ -127,6 +129,7 @@
  1072. #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
  1073. #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
  1074. #define _TIF_FOREIGN_FPSTATE (1 << TIF_FOREIGN_FPSTATE)
  1075. +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY)
  1076. #define _TIF_NOHZ (1 << TIF_NOHZ)
  1077. #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
  1078. #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
  1079. @@ -135,7 +138,9 @@
  1080. #define _TIF_32BIT (1 << TIF_32BIT)
  1081. #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
  1082. - _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE)
  1083. + _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \
  1084. + _TIF_NEED_RESCHED_LAZY)
  1085. +#define _TIF_NEED_RESCHED_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)
  1086. #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
  1087. _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
  1088. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm64/kernel/asm-offsets.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm64/kernel/asm-offsets.c
  1089. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm64/kernel/asm-offsets.c 2017-04-16 10:37:32.000000000 +0200
  1090. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm64/kernel/asm-offsets.c 2017-04-18 17:54:20.000000000 +0200
  1091. @@ -38,6 +38,7 @@
  1092. BLANK();
  1093. DEFINE(TI_FLAGS, offsetof(struct thread_info, flags));
  1094. DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count));
  1095. + DEFINE(TI_PREEMPT_LAZY, offsetof(struct thread_info, preempt_lazy_count));
  1096. DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit));
  1097. DEFINE(TI_TASK, offsetof(struct thread_info, task));
  1098. DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
  1099. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm64/kernel/entry.S linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm64/kernel/entry.S
  1100. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm64/kernel/entry.S 2017-04-16 10:37:32.000000000 +0200
  1101. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm64/kernel/entry.S 2017-04-18 17:54:20.000000000 +0200
  1102. @@ -428,11 +428,16 @@
  1103. #ifdef CONFIG_PREEMPT
  1104. ldr w24, [tsk, #TI_PREEMPT] // get preempt count
  1105. - cbnz w24, 1f // preempt count != 0
  1106. + cbnz w24, 2f // preempt count != 0
  1107. ldr x0, [tsk, #TI_FLAGS] // get flags
  1108. - tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling?
  1109. - bl el1_preempt
  1110. + tbnz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling?
  1111. +
  1112. + ldr w24, [tsk, #TI_PREEMPT_LAZY] // get preempt lazy count
  1113. + cbnz w24, 2f // preempt lazy count != 0
  1114. + tbz x0, #TIF_NEED_RESCHED_LAZY, 2f // needs rescheduling?
  1115. 1:
  1116. + bl el1_preempt
  1117. +2:
  1118. #endif
  1119. #ifdef CONFIG_TRACE_IRQFLAGS
  1120. bl trace_hardirqs_on
  1121. @@ -446,6 +451,7 @@
  1122. 1: bl preempt_schedule_irq // irq en/disable is done inside
  1123. ldr x0, [tsk, #TI_FLAGS] // get new tasks TI_FLAGS
  1124. tbnz x0, #TIF_NEED_RESCHED, 1b // needs rescheduling?
  1125. + tbnz x0, #TIF_NEED_RESCHED_LAZY, 1b // needs rescheduling?
  1126. ret x24
  1127. #endif
  1128. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm64/kernel/signal.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm64/kernel/signal.c
  1129. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/arm64/kernel/signal.c 2017-04-16 10:37:32.000000000 +0200
  1130. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/arm64/kernel/signal.c 2017-04-18 17:54:20.000000000 +0200
  1131. @@ -409,7 +409,7 @@
  1132. */
  1133. trace_hardirqs_off();
  1134. do {
  1135. - if (thread_flags & _TIF_NEED_RESCHED) {
  1136. + if (thread_flags & _TIF_NEED_RESCHED_MASK) {
  1137. schedule();
  1138. } else {
  1139. local_irq_enable();
  1140. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/mips/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/mips/Kconfig
  1141. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/mips/Kconfig 2017-04-16 10:37:33.000000000 +0200
  1142. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/mips/Kconfig 2017-04-18 17:54:20.000000000 +0200
  1143. @@ -2514,7 +2514,7 @@
  1144. #
  1145. config HIGHMEM
  1146. bool "High Memory Support"
  1147. - depends on 32BIT && CPU_SUPPORTS_HIGHMEM && SYS_SUPPORTS_HIGHMEM && !CPU_MIPS32_3_5_EVA
  1148. + depends on 32BIT && CPU_SUPPORTS_HIGHMEM && SYS_SUPPORTS_HIGHMEM && !CPU_MIPS32_3_5_EVA && !PREEMPT_RT_FULL
  1149. config CPU_SUPPORTS_HIGHMEM
  1150. bool
  1151. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/Kconfig
  1152. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/Kconfig 2017-04-16 10:37:34.000000000 +0200
  1153. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/Kconfig 2017-04-18 17:54:20.000000000 +0200
  1154. @@ -52,10 +52,11 @@
  1155. config RWSEM_GENERIC_SPINLOCK
  1156. bool
  1157. + default y if PREEMPT_RT_FULL
  1158. config RWSEM_XCHGADD_ALGORITHM
  1159. bool
  1160. - default y
  1161. + default y if !PREEMPT_RT_FULL
  1162. config GENERIC_LOCKBREAK
  1163. bool
  1164. @@ -134,6 +135,7 @@
  1165. select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
  1166. select GENERIC_STRNCPY_FROM_USER
  1167. select GENERIC_STRNLEN_USER
  1168. + select HAVE_PREEMPT_LAZY
  1169. select HAVE_MOD_ARCH_SPECIFIC
  1170. select MODULES_USE_ELF_RELA
  1171. select CLONE_BACKWARDS
  1172. @@ -321,7 +323,7 @@
  1173. config HIGHMEM
  1174. bool "High memory support"
  1175. - depends on PPC32
  1176. + depends on PPC32 && !PREEMPT_RT_FULL
  1177. source kernel/Kconfig.hz
  1178. source kernel/Kconfig.preempt
  1179. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/include/asm/thread_info.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/include/asm/thread_info.h
  1180. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/include/asm/thread_info.h 2017-04-16 10:37:35.000000000 +0200
  1181. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/include/asm/thread_info.h 2017-04-18 17:54:20.000000000 +0200
  1182. @@ -43,6 +43,8 @@
  1183. int cpu; /* cpu we're on */
  1184. int preempt_count; /* 0 => preemptable,
  1185. <0 => BUG */
  1186. + int preempt_lazy_count; /* 0 => preemptable,
  1187. + <0 => BUG */
  1188. unsigned long local_flags; /* private flags for thread */
  1189. #ifdef CONFIG_LIVEPATCH
  1190. unsigned long *livepatch_sp;
  1191. @@ -88,8 +90,7 @@
  1192. #define TIF_SYSCALL_TRACE 0 /* syscall trace active */
  1193. #define TIF_SIGPENDING 1 /* signal pending */
  1194. #define TIF_NEED_RESCHED 2 /* rescheduling necessary */
  1195. -#define TIF_POLLING_NRFLAG 3 /* true if poll_idle() is polling
  1196. - TIF_NEED_RESCHED */
  1197. +#define TIF_NEED_RESCHED_LAZY 3 /* lazy rescheduling necessary */
  1198. #define TIF_32BIT 4 /* 32 bit binary */
  1199. #define TIF_RESTORE_TM 5 /* need to restore TM FP/VEC/VSX */
  1200. #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */
  1201. @@ -107,6 +108,8 @@
  1202. #if defined(CONFIG_PPC64)
  1203. #define TIF_ELF2ABI 18 /* function descriptors must die! */
  1204. #endif
  1205. +#define TIF_POLLING_NRFLAG 19 /* true if poll_idle() is polling
  1206. + TIF_NEED_RESCHED */
  1207. /* as above, but as bit values */
  1208. #define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE)
  1209. @@ -125,14 +128,16 @@
  1210. #define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT)
  1211. #define _TIF_EMULATE_STACK_STORE (1<<TIF_EMULATE_STACK_STORE)
  1212. #define _TIF_NOHZ (1<<TIF_NOHZ)
  1213. +#define _TIF_NEED_RESCHED_LAZY (1<<TIF_NEED_RESCHED_LAZY)
  1214. #define _TIF_SYSCALL_DOTRACE (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
  1215. _TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT | \
  1216. _TIF_NOHZ)
  1217. #define _TIF_USER_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \
  1218. _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
  1219. - _TIF_RESTORE_TM)
  1220. + _TIF_RESTORE_TM | _TIF_NEED_RESCHED_LAZY)
  1221. #define _TIF_PERSYSCALL_MASK (_TIF_RESTOREALL|_TIF_NOERROR)
  1222. +#define _TIF_NEED_RESCHED_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)
  1223. /* Bits in local_flags */
  1224. /* Don't move TLF_NAPPING without adjusting the code in entry_32.S */
  1225. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kernel/asm-offsets.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kernel/asm-offsets.c
  1226. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kernel/asm-offsets.c 2017-04-16 10:37:35.000000000 +0200
  1227. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kernel/asm-offsets.c 2017-04-18 17:54:20.000000000 +0200
  1228. @@ -156,6 +156,7 @@
  1229. DEFINE(TI_FLAGS, offsetof(struct thread_info, flags));
  1230. DEFINE(TI_LOCAL_FLAGS, offsetof(struct thread_info, local_flags));
  1231. DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count));
  1232. + DEFINE(TI_PREEMPT_LAZY, offsetof(struct thread_info, preempt_lazy_count));
  1233. DEFINE(TI_TASK, offsetof(struct thread_info, task));
  1234. DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
  1235. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kernel/entry_32.S linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kernel/entry_32.S
  1236. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kernel/entry_32.S 2017-04-16 10:37:35.000000000 +0200
  1237. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kernel/entry_32.S 2017-04-18 17:54:20.000000000 +0200
  1238. @@ -835,7 +835,14 @@
  1239. cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
  1240. bne restore
  1241. andi. r8,r8,_TIF_NEED_RESCHED
  1242. + bne+ 1f
  1243. + lwz r0,TI_PREEMPT_LAZY(r9)
  1244. + cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
  1245. + bne restore
  1246. + lwz r0,TI_FLAGS(r9)
  1247. + andi. r0,r0,_TIF_NEED_RESCHED_LAZY
  1248. beq+ restore
  1249. +1:
  1250. lwz r3,_MSR(r1)
  1251. andi. r0,r3,MSR_EE /* interrupts off? */
  1252. beq restore /* don't schedule if so */
  1253. @@ -846,11 +853,11 @@
  1254. */
  1255. bl trace_hardirqs_off
  1256. #endif
  1257. -1: bl preempt_schedule_irq
  1258. +2: bl preempt_schedule_irq
  1259. CURRENT_THREAD_INFO(r9, r1)
  1260. lwz r3,TI_FLAGS(r9)
  1261. - andi. r0,r3,_TIF_NEED_RESCHED
  1262. - bne- 1b
  1263. + andi. r0,r3,_TIF_NEED_RESCHED_MASK
  1264. + bne- 2b
  1265. #ifdef CONFIG_TRACE_IRQFLAGS
  1266. /* And now, to properly rebalance the above, we tell lockdep they
  1267. * are being turned back on, which will happen when we return
  1268. @@ -1171,7 +1178,7 @@
  1269. #endif /* !(CONFIG_4xx || CONFIG_BOOKE) */
  1270. do_work: /* r10 contains MSR_KERNEL here */
  1271. - andi. r0,r9,_TIF_NEED_RESCHED
  1272. + andi. r0,r9,_TIF_NEED_RESCHED_MASK
  1273. beq do_user_signal
  1274. do_resched: /* r10 contains MSR_KERNEL here */
  1275. @@ -1192,7 +1199,7 @@
  1276. MTMSRD(r10) /* disable interrupts */
  1277. CURRENT_THREAD_INFO(r9, r1)
  1278. lwz r9,TI_FLAGS(r9)
  1279. - andi. r0,r9,_TIF_NEED_RESCHED
  1280. + andi. r0,r9,_TIF_NEED_RESCHED_MASK
  1281. bne- do_resched
  1282. andi. r0,r9,_TIF_USER_WORK_MASK
  1283. beq restore_user
  1284. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kernel/entry_64.S linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kernel/entry_64.S
  1285. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kernel/entry_64.S 2017-04-16 10:37:35.000000000 +0200
  1286. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kernel/entry_64.S 2017-04-18 17:54:20.000000000 +0200
  1287. @@ -656,7 +656,7 @@
  1288. bl restore_math
  1289. b restore
  1290. #endif
  1291. -1: andi. r0,r4,_TIF_NEED_RESCHED
  1292. +1: andi. r0,r4,_TIF_NEED_RESCHED_MASK
  1293. beq 2f
  1294. bl restore_interrupts
  1295. SCHEDULE_USER
  1296. @@ -718,10 +718,18 @@
  1297. #ifdef CONFIG_PREEMPT
  1298. /* Check if we need to preempt */
  1299. + lwz r8,TI_PREEMPT(r9)
  1300. + cmpwi 0,r8,0 /* if non-zero, just restore regs and return */
  1301. + bne restore
  1302. andi. r0,r4,_TIF_NEED_RESCHED
  1303. + bne+ check_count
  1304. +
  1305. + andi. r0,r4,_TIF_NEED_RESCHED_LAZY
  1306. beq+ restore
  1307. + lwz r8,TI_PREEMPT_LAZY(r9)
  1308. +
  1309. /* Check that preempt_count() == 0 and interrupts are enabled */
  1310. - lwz r8,TI_PREEMPT(r9)
  1311. +check_count:
  1312. cmpwi cr1,r8,0
  1313. ld r0,SOFTE(r1)
  1314. cmpdi r0,0
  1315. @@ -738,7 +746,7 @@
  1316. /* Re-test flags and eventually loop */
  1317. CURRENT_THREAD_INFO(r9, r1)
  1318. ld r4,TI_FLAGS(r9)
  1319. - andi. r0,r4,_TIF_NEED_RESCHED
  1320. + andi. r0,r4,_TIF_NEED_RESCHED_MASK
  1321. bne 1b
  1322. /*
  1323. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kernel/irq.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kernel/irq.c
  1324. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kernel/irq.c 2017-04-16 10:37:35.000000000 +0200
  1325. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kernel/irq.c 2017-04-18 17:54:20.000000000 +0200
  1326. @@ -638,6 +638,7 @@
  1327. }
  1328. }
  1329. +#ifndef CONFIG_PREEMPT_RT_FULL
  1330. void do_softirq_own_stack(void)
  1331. {
  1332. struct thread_info *curtp, *irqtp;
  1333. @@ -655,6 +656,7 @@
  1334. if (irqtp->flags)
  1335. set_bits(irqtp->flags, &curtp->flags);
  1336. }
  1337. +#endif
  1338. irq_hw_number_t virq_to_hw(unsigned int virq)
  1339. {
  1340. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kernel/misc_32.S linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kernel/misc_32.S
  1341. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kernel/misc_32.S 2017-04-16 10:37:35.000000000 +0200
  1342. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kernel/misc_32.S 2017-04-18 17:54:20.000000000 +0200
  1343. @@ -41,6 +41,7 @@
  1344. * We store the saved ksp_limit in the unused part
  1345. * of the STACK_FRAME_OVERHEAD
  1346. */
  1347. +#ifndef CONFIG_PREEMPT_RT_FULL
  1348. _GLOBAL(call_do_softirq)
  1349. mflr r0
  1350. stw r0,4(r1)
  1351. @@ -57,6 +58,7 @@
  1352. stw r10,THREAD+KSP_LIMIT(r2)
  1353. mtlr r0
  1354. blr
  1355. +#endif
  1356. /*
  1357. * void call_do_irq(struct pt_regs *regs, struct thread_info *irqtp);
  1358. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kernel/misc_64.S linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kernel/misc_64.S
  1359. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kernel/misc_64.S 2017-04-16 10:37:35.000000000 +0200
  1360. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kernel/misc_64.S 2017-04-18 17:54:20.000000000 +0200
  1361. @@ -31,6 +31,7 @@
  1362. .text
  1363. +#ifndef CONFIG_PREEMPT_RT_FULL
  1364. _GLOBAL(call_do_softirq)
  1365. mflr r0
  1366. std r0,16(r1)
  1367. @@ -41,6 +42,7 @@
  1368. ld r0,16(r1)
  1369. mtlr r0
  1370. blr
  1371. +#endif
  1372. _GLOBAL(call_do_irq)
  1373. mflr r0
  1374. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kvm/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kvm/Kconfig
  1375. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/kvm/Kconfig 2017-04-16 10:37:35.000000000 +0200
  1376. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/kvm/Kconfig 2017-04-18 17:54:20.000000000 +0200
  1377. @@ -175,6 +175,7 @@
  1378. config KVM_MPIC
  1379. bool "KVM in-kernel MPIC emulation"
  1380. depends on KVM && E500
  1381. + depends on !PREEMPT_RT_FULL
  1382. select HAVE_KVM_IRQCHIP
  1383. select HAVE_KVM_IRQFD
  1384. select HAVE_KVM_IRQ_ROUTING
  1385. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/platforms/ps3/device-init.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/platforms/ps3/device-init.c
  1386. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/powerpc/platforms/ps3/device-init.c 2017-04-16 10:37:35.000000000 +0200
  1387. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/powerpc/platforms/ps3/device-init.c 2017-04-18 17:54:20.000000000 +0200
  1388. @@ -752,7 +752,7 @@
  1389. }
  1390. pr_debug("%s:%u: notification %s issued\n", __func__, __LINE__, op);
  1391. - res = wait_event_interruptible(dev->done.wait,
  1392. + res = swait_event_interruptible(dev->done.wait,
  1393. dev->done.done || kthread_should_stop());
  1394. if (kthread_should_stop())
  1395. res = -EINTR;
  1396. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/sh/kernel/irq.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/sh/kernel/irq.c
  1397. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/sh/kernel/irq.c 2017-04-16 10:37:36.000000000 +0200
  1398. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/sh/kernel/irq.c 2017-04-18 17:54:20.000000000 +0200
  1399. @@ -147,6 +147,7 @@
  1400. hardirq_ctx[cpu] = NULL;
  1401. }
  1402. +#ifndef CONFIG_PREEMPT_RT_FULL
  1403. void do_softirq_own_stack(void)
  1404. {
  1405. struct thread_info *curctx;
  1406. @@ -174,6 +175,7 @@
  1407. "r5", "r6", "r7", "r8", "r9", "r15", "t", "pr"
  1408. );
  1409. }
  1410. +#endif
  1411. #else
  1412. static inline void handle_one_irq(unsigned int irq)
  1413. {
  1414. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/sparc/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/sparc/Kconfig
  1415. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/sparc/Kconfig 2017-04-16 10:37:36.000000000 +0200
  1416. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/sparc/Kconfig 2017-04-18 17:54:21.000000000 +0200
  1417. @@ -194,12 +194,10 @@
  1418. source kernel/Kconfig.hz
  1419. config RWSEM_GENERIC_SPINLOCK
  1420. - bool
  1421. - default y if SPARC32
  1422. + def_bool PREEMPT_RT_FULL
  1423. config RWSEM_XCHGADD_ALGORITHM
  1424. - bool
  1425. - default y if SPARC64
  1426. + def_bool !RWSEM_GENERIC_SPINLOCK && !PREEMPT_RT_FULL
  1427. config GENERIC_HWEIGHT
  1428. bool
  1429. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/sparc/kernel/irq_64.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/sparc/kernel/irq_64.c
  1430. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/sparc/kernel/irq_64.c 2017-04-16 10:37:36.000000000 +0200
  1431. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/sparc/kernel/irq_64.c 2017-04-18 17:54:21.000000000 +0200
  1432. @@ -854,6 +854,7 @@
  1433. set_irq_regs(old_regs);
  1434. }
  1435. +#ifndef CONFIG_PREEMPT_RT_FULL
  1436. void do_softirq_own_stack(void)
  1437. {
  1438. void *orig_sp, *sp = softirq_stack[smp_processor_id()];
  1439. @@ -868,6 +869,7 @@
  1440. __asm__ __volatile__("mov %0, %%sp"
  1441. : : "r" (orig_sp));
  1442. }
  1443. +#endif
  1444. #ifdef CONFIG_HOTPLUG_CPU
  1445. void fixup_irqs(void)
  1446. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/Kconfig
  1447. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/Kconfig 2017-04-16 10:37:37.000000000 +0200
  1448. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/Kconfig 2017-04-18 17:54:21.000000000 +0200
  1449. @@ -17,6 +17,7 @@
  1450. ### Arch settings
  1451. config X86
  1452. def_bool y
  1453. + select HAVE_PREEMPT_LAZY
  1454. select ACPI_LEGACY_TABLES_LOOKUP if ACPI
  1455. select ACPI_SYSTEM_POWER_STATES_SUPPORT if ACPI
  1456. select ANON_INODES
  1457. @@ -232,8 +233,11 @@
  1458. def_bool y
  1459. depends on ISA_DMA_API
  1460. +config RWSEM_GENERIC_SPINLOCK
  1461. + def_bool PREEMPT_RT_FULL
  1462. +
  1463. config RWSEM_XCHGADD_ALGORITHM
  1464. - def_bool y
  1465. + def_bool !RWSEM_GENERIC_SPINLOCK && !PREEMPT_RT_FULL
  1466. config GENERIC_CALIBRATE_DELAY
  1467. def_bool y
  1468. @@ -897,7 +901,7 @@
  1469. config MAXSMP
  1470. bool "Enable Maximum number of SMP Processors and NUMA Nodes"
  1471. depends on X86_64 && SMP && DEBUG_KERNEL
  1472. - select CPUMASK_OFFSTACK
  1473. + select CPUMASK_OFFSTACK if !PREEMPT_RT_FULL
  1474. ---help---
  1475. Enable maximum number of CPUS and NUMA Nodes for this architecture.
  1476. If unsure, say N.
  1477. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/crypto/aesni-intel_glue.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/crypto/aesni-intel_glue.c
  1478. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/crypto/aesni-intel_glue.c 2017-04-16 10:37:37.000000000 +0200
  1479. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/crypto/aesni-intel_glue.c 2017-04-18 17:54:21.000000000 +0200
  1480. @@ -372,14 +372,14 @@
  1481. err = blkcipher_walk_virt(desc, &walk);
  1482. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1483. - kernel_fpu_begin();
  1484. while ((nbytes = walk.nbytes)) {
  1485. + kernel_fpu_begin();
  1486. aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
  1487. - nbytes & AES_BLOCK_MASK);
  1488. + nbytes & AES_BLOCK_MASK);
  1489. + kernel_fpu_end();
  1490. nbytes &= AES_BLOCK_SIZE - 1;
  1491. err = blkcipher_walk_done(desc, &walk, nbytes);
  1492. }
  1493. - kernel_fpu_end();
  1494. return err;
  1495. }
  1496. @@ -396,14 +396,14 @@
  1497. err = blkcipher_walk_virt(desc, &walk);
  1498. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1499. - kernel_fpu_begin();
  1500. while ((nbytes = walk.nbytes)) {
  1501. + kernel_fpu_begin();
  1502. aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
  1503. nbytes & AES_BLOCK_MASK);
  1504. + kernel_fpu_end();
  1505. nbytes &= AES_BLOCK_SIZE - 1;
  1506. err = blkcipher_walk_done(desc, &walk, nbytes);
  1507. }
  1508. - kernel_fpu_end();
  1509. return err;
  1510. }
  1511. @@ -420,14 +420,14 @@
  1512. err = blkcipher_walk_virt(desc, &walk);
  1513. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1514. - kernel_fpu_begin();
  1515. while ((nbytes = walk.nbytes)) {
  1516. + kernel_fpu_begin();
  1517. aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
  1518. nbytes & AES_BLOCK_MASK, walk.iv);
  1519. + kernel_fpu_end();
  1520. nbytes &= AES_BLOCK_SIZE - 1;
  1521. err = blkcipher_walk_done(desc, &walk, nbytes);
  1522. }
  1523. - kernel_fpu_end();
  1524. return err;
  1525. }
  1526. @@ -444,14 +444,14 @@
  1527. err = blkcipher_walk_virt(desc, &walk);
  1528. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1529. - kernel_fpu_begin();
  1530. while ((nbytes = walk.nbytes)) {
  1531. + kernel_fpu_begin();
  1532. aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
  1533. nbytes & AES_BLOCK_MASK, walk.iv);
  1534. + kernel_fpu_end();
  1535. nbytes &= AES_BLOCK_SIZE - 1;
  1536. err = blkcipher_walk_done(desc, &walk, nbytes);
  1537. }
  1538. - kernel_fpu_end();
  1539. return err;
  1540. }
  1541. @@ -503,18 +503,20 @@
  1542. err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
  1543. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1544. - kernel_fpu_begin();
  1545. while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
  1546. + kernel_fpu_begin();
  1547. aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
  1548. nbytes & AES_BLOCK_MASK, walk.iv);
  1549. + kernel_fpu_end();
  1550. nbytes &= AES_BLOCK_SIZE - 1;
  1551. err = blkcipher_walk_done(desc, &walk, nbytes);
  1552. }
  1553. if (walk.nbytes) {
  1554. + kernel_fpu_begin();
  1555. ctr_crypt_final(ctx, &walk);
  1556. + kernel_fpu_end();
  1557. err = blkcipher_walk_done(desc, &walk, 0);
  1558. }
  1559. - kernel_fpu_end();
  1560. return err;
  1561. }
  1562. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/crypto/cast5_avx_glue.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/crypto/cast5_avx_glue.c
  1563. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/crypto/cast5_avx_glue.c 2017-04-16 10:37:37.000000000 +0200
  1564. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/crypto/cast5_avx_glue.c 2017-04-18 17:54:21.000000000 +0200
  1565. @@ -59,7 +59,7 @@
  1566. static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
  1567. bool enc)
  1568. {
  1569. - bool fpu_enabled = false;
  1570. + bool fpu_enabled;
  1571. struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  1572. const unsigned int bsize = CAST5_BLOCK_SIZE;
  1573. unsigned int nbytes;
  1574. @@ -75,7 +75,7 @@
  1575. u8 *wsrc = walk->src.virt.addr;
  1576. u8 *wdst = walk->dst.virt.addr;
  1577. - fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
  1578. + fpu_enabled = cast5_fpu_begin(false, nbytes);
  1579. /* Process multi-block batch */
  1580. if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
  1581. @@ -103,10 +103,9 @@
  1582. } while (nbytes >= bsize);
  1583. done:
  1584. + cast5_fpu_end(fpu_enabled);
  1585. err = blkcipher_walk_done(desc, walk, nbytes);
  1586. }
  1587. -
  1588. - cast5_fpu_end(fpu_enabled);
  1589. return err;
  1590. }
  1591. @@ -227,7 +226,7 @@
  1592. static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  1593. struct scatterlist *src, unsigned int nbytes)
  1594. {
  1595. - bool fpu_enabled = false;
  1596. + bool fpu_enabled;
  1597. struct blkcipher_walk walk;
  1598. int err;
  1599. @@ -236,12 +235,11 @@
  1600. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1601. while ((nbytes = walk.nbytes)) {
  1602. - fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
  1603. + fpu_enabled = cast5_fpu_begin(false, nbytes);
  1604. nbytes = __cbc_decrypt(desc, &walk);
  1605. + cast5_fpu_end(fpu_enabled);
  1606. err = blkcipher_walk_done(desc, &walk, nbytes);
  1607. }
  1608. -
  1609. - cast5_fpu_end(fpu_enabled);
  1610. return err;
  1611. }
  1612. @@ -311,7 +309,7 @@
  1613. static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  1614. struct scatterlist *src, unsigned int nbytes)
  1615. {
  1616. - bool fpu_enabled = false;
  1617. + bool fpu_enabled;
  1618. struct blkcipher_walk walk;
  1619. int err;
  1620. @@ -320,13 +318,12 @@
  1621. desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  1622. while ((nbytes = walk.nbytes) >= CAST5_BLOCK_SIZE) {
  1623. - fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
  1624. + fpu_enabled = cast5_fpu_begin(false, nbytes);
  1625. nbytes = __ctr_crypt(desc, &walk);
  1626. + cast5_fpu_end(fpu_enabled);
  1627. err = blkcipher_walk_done(desc, &walk, nbytes);
  1628. }
  1629. - cast5_fpu_end(fpu_enabled);
  1630. -
  1631. if (walk.nbytes) {
  1632. ctr_crypt_final(desc, &walk);
  1633. err = blkcipher_walk_done(desc, &walk, 0);
  1634. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/crypto/glue_helper.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/crypto/glue_helper.c
  1635. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/crypto/glue_helper.c 2017-04-16 10:37:37.000000000 +0200
  1636. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/crypto/glue_helper.c 2017-04-18 17:54:21.000000000 +0200
  1637. @@ -39,7 +39,7 @@
  1638. void *ctx = crypto_blkcipher_ctx(desc->tfm);
  1639. const unsigned int bsize = 128 / 8;
  1640. unsigned int nbytes, i, func_bytes;
  1641. - bool fpu_enabled = false;
  1642. + bool fpu_enabled;
  1643. int err;
  1644. err = blkcipher_walk_virt(desc, walk);
  1645. @@ -49,7 +49,7 @@
  1646. u8 *wdst = walk->dst.virt.addr;
  1647. fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
  1648. - desc, fpu_enabled, nbytes);
  1649. + desc, false, nbytes);
  1650. for (i = 0; i < gctx->num_funcs; i++) {
  1651. func_bytes = bsize * gctx->funcs[i].num_blocks;
  1652. @@ -71,10 +71,10 @@
  1653. }
  1654. done:
  1655. + glue_fpu_end(fpu_enabled);
  1656. err = blkcipher_walk_done(desc, walk, nbytes);
  1657. }
  1658. - glue_fpu_end(fpu_enabled);
  1659. return err;
  1660. }
  1661. @@ -194,7 +194,7 @@
  1662. struct scatterlist *src, unsigned int nbytes)
  1663. {
  1664. const unsigned int bsize = 128 / 8;
  1665. - bool fpu_enabled = false;
  1666. + bool fpu_enabled;
  1667. struct blkcipher_walk walk;
  1668. int err;
  1669. @@ -203,12 +203,12 @@
  1670. while ((nbytes = walk.nbytes)) {
  1671. fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
  1672. - desc, fpu_enabled, nbytes);
  1673. + desc, false, nbytes);
  1674. nbytes = __glue_cbc_decrypt_128bit(gctx, desc, &walk);
  1675. + glue_fpu_end(fpu_enabled);
  1676. err = blkcipher_walk_done(desc, &walk, nbytes);
  1677. }
  1678. - glue_fpu_end(fpu_enabled);
  1679. return err;
  1680. }
  1681. EXPORT_SYMBOL_GPL(glue_cbc_decrypt_128bit);
  1682. @@ -277,7 +277,7 @@
  1683. struct scatterlist *src, unsigned int nbytes)
  1684. {
  1685. const unsigned int bsize = 128 / 8;
  1686. - bool fpu_enabled = false;
  1687. + bool fpu_enabled;
  1688. struct blkcipher_walk walk;
  1689. int err;
  1690. @@ -286,13 +286,12 @@
  1691. while ((nbytes = walk.nbytes) >= bsize) {
  1692. fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
  1693. - desc, fpu_enabled, nbytes);
  1694. + desc, false, nbytes);
  1695. nbytes = __glue_ctr_crypt_128bit(gctx, desc, &walk);
  1696. + glue_fpu_end(fpu_enabled);
  1697. err = blkcipher_walk_done(desc, &walk, nbytes);
  1698. }
  1699. - glue_fpu_end(fpu_enabled);
  1700. -
  1701. if (walk.nbytes) {
  1702. glue_ctr_crypt_final_128bit(
  1703. gctx->funcs[gctx->num_funcs - 1].fn_u.ctr, desc, &walk);
  1704. @@ -347,7 +346,7 @@
  1705. void *tweak_ctx, void *crypt_ctx)
  1706. {
  1707. const unsigned int bsize = 128 / 8;
  1708. - bool fpu_enabled = false;
  1709. + bool fpu_enabled;
  1710. struct blkcipher_walk walk;
  1711. int err;
  1712. @@ -360,21 +359,21 @@
  1713. /* set minimum length to bsize, for tweak_fn */
  1714. fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
  1715. - desc, fpu_enabled,
  1716. + desc, false,
  1717. nbytes < bsize ? bsize : nbytes);
  1718. -
  1719. /* calculate first value of T */
  1720. tweak_fn(tweak_ctx, walk.iv, walk.iv);
  1721. + glue_fpu_end(fpu_enabled);
  1722. while (nbytes) {
  1723. + fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
  1724. + desc, false, nbytes);
  1725. nbytes = __glue_xts_crypt_128bit(gctx, crypt_ctx, desc, &walk);
  1726. + glue_fpu_end(fpu_enabled);
  1727. err = blkcipher_walk_done(desc, &walk, nbytes);
  1728. nbytes = walk.nbytes;
  1729. }
  1730. -
  1731. - glue_fpu_end(fpu_enabled);
  1732. -
  1733. return err;
  1734. }
  1735. EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit);
  1736. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/entry/common.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/entry/common.c
  1737. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/entry/common.c 2017-04-16 10:37:37.000000000 +0200
  1738. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/entry/common.c 2017-04-18 17:54:21.000000000 +0200
  1739. @@ -129,7 +129,7 @@
  1740. #define EXIT_TO_USERMODE_LOOP_FLAGS \
  1741. (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
  1742. - _TIF_NEED_RESCHED | _TIF_USER_RETURN_NOTIFY)
  1743. + _TIF_NEED_RESCHED_MASK | _TIF_USER_RETURN_NOTIFY)
  1744. static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
  1745. {
  1746. @@ -145,9 +145,16 @@
  1747. /* We have work to do. */
  1748. local_irq_enable();
  1749. - if (cached_flags & _TIF_NEED_RESCHED)
  1750. + if (cached_flags & _TIF_NEED_RESCHED_MASK)
  1751. schedule();
  1752. +#ifdef ARCH_RT_DELAYS_SIGNAL_SEND
  1753. + if (unlikely(current->forced_info.si_signo)) {
  1754. + struct task_struct *t = current;
  1755. + force_sig_info(t->forced_info.si_signo, &t->forced_info, t);
  1756. + t->forced_info.si_signo = 0;
  1757. + }
  1758. +#endif
  1759. if (cached_flags & _TIF_UPROBE)
  1760. uprobe_notify_resume(regs);
  1761. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/entry/entry_32.S linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/entry/entry_32.S
  1762. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/entry/entry_32.S 2017-04-16 10:37:37.000000000 +0200
  1763. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/entry/entry_32.S 2017-04-18 17:54:21.000000000 +0200
  1764. @@ -308,8 +308,25 @@
  1765. ENTRY(resume_kernel)
  1766. DISABLE_INTERRUPTS(CLBR_ANY)
  1767. need_resched:
  1768. + # preempt count == 0 + NEED_RS set?
  1769. cmpl $0, PER_CPU_VAR(__preempt_count)
  1770. +#ifndef CONFIG_PREEMPT_LAZY
  1771. jnz restore_all
  1772. +#else
  1773. + jz test_int_off
  1774. +
  1775. + # atleast preempt count == 0 ?
  1776. + cmpl $_PREEMPT_ENABLED,PER_CPU_VAR(__preempt_count)
  1777. + jne restore_all
  1778. +
  1779. + movl PER_CPU_VAR(current_task), %ebp
  1780. + cmpl $0,TASK_TI_preempt_lazy_count(%ebp) # non-zero preempt_lazy_count ?
  1781. + jnz restore_all
  1782. +
  1783. + testl $_TIF_NEED_RESCHED_LAZY, TASK_TI_flags(%ebp)
  1784. + jz restore_all
  1785. +test_int_off:
  1786. +#endif
  1787. testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ?
  1788. jz restore_all
  1789. call preempt_schedule_irq
  1790. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/entry/entry_64.S linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/entry/entry_64.S
  1791. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/entry/entry_64.S 2017-04-16 10:37:37.000000000 +0200
  1792. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/entry/entry_64.S 2017-04-18 17:54:21.000000000 +0200
  1793. @@ -546,7 +546,23 @@
  1794. bt $9, EFLAGS(%rsp) /* were interrupts off? */
  1795. jnc 1f
  1796. 0: cmpl $0, PER_CPU_VAR(__preempt_count)
  1797. +#ifndef CONFIG_PREEMPT_LAZY
  1798. jnz 1f
  1799. +#else
  1800. + jz do_preempt_schedule_irq
  1801. +
  1802. + # atleast preempt count == 0 ?
  1803. + cmpl $_PREEMPT_ENABLED,PER_CPU_VAR(__preempt_count)
  1804. + jnz 1f
  1805. +
  1806. + movq PER_CPU_VAR(current_task), %rcx
  1807. + cmpl $0, TASK_TI_preempt_lazy_count(%rcx)
  1808. + jnz 1f
  1809. +
  1810. + bt $TIF_NEED_RESCHED_LAZY,TASK_TI_flags(%rcx)
  1811. + jnc 1f
  1812. +do_preempt_schedule_irq:
  1813. +#endif
  1814. call preempt_schedule_irq
  1815. jmp 0b
  1816. 1:
  1817. @@ -894,6 +910,7 @@
  1818. jmp 2b
  1819. .previous
  1820. +#ifndef CONFIG_PREEMPT_RT_FULL
  1821. /* Call softirq on interrupt stack. Interrupts are off. */
  1822. ENTRY(do_softirq_own_stack)
  1823. pushq %rbp
  1824. @@ -906,6 +923,7 @@
  1825. decl PER_CPU_VAR(irq_count)
  1826. ret
  1827. END(do_softirq_own_stack)
  1828. +#endif
  1829. #ifdef CONFIG_XEN
  1830. idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
  1831. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/include/asm/preempt.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/include/asm/preempt.h
  1832. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/include/asm/preempt.h 2017-04-16 10:37:37.000000000 +0200
  1833. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/include/asm/preempt.h 2017-04-18 17:54:21.000000000 +0200
  1834. @@ -79,17 +79,46 @@
  1835. * a decrement which hits zero means we have no preempt_count and should
  1836. * reschedule.
  1837. */
  1838. -static __always_inline bool __preempt_count_dec_and_test(void)
  1839. +static __always_inline bool ____preempt_count_dec_and_test(void)
  1840. {
  1841. GEN_UNARY_RMWcc("decl", __preempt_count, __percpu_arg(0), e);
  1842. }
  1843. +static __always_inline bool __preempt_count_dec_and_test(void)
  1844. +{
  1845. + if (____preempt_count_dec_and_test())
  1846. + return true;
  1847. +#ifdef CONFIG_PREEMPT_LAZY
  1848. + if (current_thread_info()->preempt_lazy_count)
  1849. + return false;
  1850. + return test_thread_flag(TIF_NEED_RESCHED_LAZY);
  1851. +#else
  1852. + return false;
  1853. +#endif
  1854. +}
  1855. +
  1856. /*
  1857. * Returns true when we need to resched and can (barring IRQ state).
  1858. */
  1859. static __always_inline bool should_resched(int preempt_offset)
  1860. {
  1861. +#ifdef CONFIG_PREEMPT_LAZY
  1862. + u32 tmp;
  1863. +
  1864. + tmp = raw_cpu_read_4(__preempt_count);
  1865. + if (tmp == preempt_offset)
  1866. + return true;
  1867. +
  1868. + /* preempt count == 0 ? */
  1869. + tmp &= ~PREEMPT_NEED_RESCHED;
  1870. + if (tmp)
  1871. + return false;
  1872. + if (current_thread_info()->preempt_lazy_count)
  1873. + return false;
  1874. + return test_thread_flag(TIF_NEED_RESCHED_LAZY);
  1875. +#else
  1876. return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
  1877. +#endif
  1878. }
  1879. #ifdef CONFIG_PREEMPT
  1880. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/include/asm/signal.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/include/asm/signal.h
  1881. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/include/asm/signal.h 2017-04-16 10:37:37.000000000 +0200
  1882. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/include/asm/signal.h 2017-04-18 17:54:21.000000000 +0200
  1883. @@ -27,6 +27,19 @@
  1884. #define SA_IA32_ABI 0x02000000u
  1885. #define SA_X32_ABI 0x01000000u
  1886. +/*
  1887. + * Because some traps use the IST stack, we must keep preemption
  1888. + * disabled while calling do_trap(), but do_trap() may call
  1889. + * force_sig_info() which will grab the signal spin_locks for the
  1890. + * task, which in PREEMPT_RT_FULL are mutexes. By defining
  1891. + * ARCH_RT_DELAYS_SIGNAL_SEND the force_sig_info() will set
  1892. + * TIF_NOTIFY_RESUME and set up the signal to be sent on exit of the
  1893. + * trap.
  1894. + */
  1895. +#if defined(CONFIG_PREEMPT_RT_FULL)
  1896. +#define ARCH_RT_DELAYS_SIGNAL_SEND
  1897. +#endif
  1898. +
  1899. #ifndef CONFIG_COMPAT
  1900. typedef sigset_t compat_sigset_t;
  1901. #endif
  1902. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/include/asm/stackprotector.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/include/asm/stackprotector.h
  1903. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/include/asm/stackprotector.h 2017-04-16 10:37:37.000000000 +0200
  1904. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/include/asm/stackprotector.h 2017-04-18 17:54:21.000000000 +0200
  1905. @@ -59,7 +59,7 @@
  1906. */
  1907. static __always_inline void boot_init_stack_canary(void)
  1908. {
  1909. - u64 canary;
  1910. + u64 uninitialized_var(canary);
  1911. u64 tsc;
  1912. #ifdef CONFIG_X86_64
  1913. @@ -70,8 +70,15 @@
  1914. * of randomness. The TSC only matters for very early init,
  1915. * there it already has some randomness on most systems. Later
  1916. * on during the bootup the random pool has true entropy too.
  1917. + *
  1918. + * For preempt-rt we need to weaken the randomness a bit, as
  1919. + * we can't call into the random generator from atomic context
  1920. + * due to locking constraints. We just leave canary
  1921. + * uninitialized and use the TSC based randomness on top of it.
  1922. */
  1923. +#ifndef CONFIG_PREEMPT_RT_FULL
  1924. get_random_bytes(&canary, sizeof(canary));
  1925. +#endif
  1926. tsc = rdtsc();
  1927. canary += tsc + (tsc << 32UL);
  1928. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/include/asm/thread_info.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/include/asm/thread_info.h
  1929. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/include/asm/thread_info.h 2017-04-16 10:37:37.000000000 +0200
  1930. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/include/asm/thread_info.h 2017-04-18 17:54:21.000000000 +0200
  1931. @@ -54,11 +54,14 @@
  1932. struct thread_info {
  1933. unsigned long flags; /* low level flags */
  1934. + int preempt_lazy_count; /* 0 => lazy preemptable
  1935. + <0 => BUG */
  1936. };
  1937. #define INIT_THREAD_INFO(tsk) \
  1938. { \
  1939. .flags = 0, \
  1940. + .preempt_lazy_count = 0, \
  1941. }
  1942. #define init_stack (init_thread_union.stack)
  1943. @@ -67,6 +70,10 @@
  1944. #include <asm/asm-offsets.h>
  1945. +#define GET_THREAD_INFO(reg) \
  1946. + _ASM_MOV PER_CPU_VAR(cpu_current_top_of_stack),reg ; \
  1947. + _ASM_SUB $(THREAD_SIZE),reg ;
  1948. +
  1949. #endif
  1950. /*
  1951. @@ -85,6 +92,7 @@
  1952. #define TIF_SYSCALL_EMU 6 /* syscall emulation active */
  1953. #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */
  1954. #define TIF_SECCOMP 8 /* secure computing */
  1955. +#define TIF_NEED_RESCHED_LAZY 9 /* lazy rescheduling necessary */
  1956. #define TIF_USER_RETURN_NOTIFY 11 /* notify kernel of userspace return */
  1957. #define TIF_UPROBE 12 /* breakpointed or singlestepping */
  1958. #define TIF_NOTSC 16 /* TSC is not accessible in userland */
  1959. @@ -108,6 +116,7 @@
  1960. #define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU)
  1961. #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
  1962. #define _TIF_SECCOMP (1 << TIF_SECCOMP)
  1963. +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY)
  1964. #define _TIF_USER_RETURN_NOTIFY (1 << TIF_USER_RETURN_NOTIFY)
  1965. #define _TIF_UPROBE (1 << TIF_UPROBE)
  1966. #define _TIF_NOTSC (1 << TIF_NOTSC)
  1967. @@ -143,6 +152,8 @@
  1968. #define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY)
  1969. #define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW)
  1970. +#define _TIF_NEED_RESCHED_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)
  1971. +
  1972. #define STACK_WARN (THREAD_SIZE/8)
  1973. /*
  1974. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/include/asm/uv/uv_bau.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/include/asm/uv/uv_bau.h
  1975. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/include/asm/uv/uv_bau.h 2017-04-16 10:37:37.000000000 +0200
  1976. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/include/asm/uv/uv_bau.h 2017-04-18 17:54:21.000000000 +0200
  1977. @@ -624,9 +624,9 @@
  1978. cycles_t send_message;
  1979. cycles_t period_end;
  1980. cycles_t period_time;
  1981. - spinlock_t uvhub_lock;
  1982. - spinlock_t queue_lock;
  1983. - spinlock_t disable_lock;
  1984. + raw_spinlock_t uvhub_lock;
  1985. + raw_spinlock_t queue_lock;
  1986. + raw_spinlock_t disable_lock;
  1987. /* tunables */
  1988. int max_concurr;
  1989. int max_concurr_const;
  1990. @@ -815,15 +815,15 @@
  1991. * to be lowered below the current 'v'. atomic_add_unless can only stop
  1992. * on equal.
  1993. */
  1994. -static inline int atomic_inc_unless_ge(spinlock_t *lock, atomic_t *v, int u)
  1995. +static inline int atomic_inc_unless_ge(raw_spinlock_t *lock, atomic_t *v, int u)
  1996. {
  1997. - spin_lock(lock);
  1998. + raw_spin_lock(lock);
  1999. if (atomic_read(v) >= u) {
  2000. - spin_unlock(lock);
  2001. + raw_spin_unlock(lock);
  2002. return 0;
  2003. }
  2004. atomic_inc(v);
  2005. - spin_unlock(lock);
  2006. + raw_spin_unlock(lock);
  2007. return 1;
  2008. }
  2009. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kernel/acpi/boot.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kernel/acpi/boot.c
  2010. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kernel/acpi/boot.c 2017-04-16 10:37:37.000000000 +0200
  2011. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kernel/acpi/boot.c 2017-04-18 17:54:21.000000000 +0200
  2012. @@ -87,7 +87,9 @@
  2013. * ->ioapic_mutex
  2014. * ->ioapic_lock
  2015. */
  2016. +#ifdef CONFIG_X86_IO_APIC
  2017. static DEFINE_MUTEX(acpi_ioapic_lock);
  2018. +#endif
  2019. /* --------------------------------------------------------------------------
  2020. Boot-time Configuration
  2021. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kernel/apic/io_apic.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kernel/apic/io_apic.c
  2022. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kernel/apic/io_apic.c 2017-04-16 10:37:37.000000000 +0200
  2023. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kernel/apic/io_apic.c 2017-04-18 17:54:21.000000000 +0200
  2024. @@ -1712,7 +1712,8 @@
  2025. static inline bool ioapic_irqd_mask(struct irq_data *data)
  2026. {
  2027. /* If we are moving the irq we need to mask it */
  2028. - if (unlikely(irqd_is_setaffinity_pending(data))) {
  2029. + if (unlikely(irqd_is_setaffinity_pending(data) &&
  2030. + !irqd_irq_inprogress(data))) {
  2031. mask_ioapic_irq(data);
  2032. return true;
  2033. }
  2034. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kernel/asm-offsets.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kernel/asm-offsets.c
  2035. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kernel/asm-offsets.c 2017-04-16 10:37:37.000000000 +0200
  2036. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kernel/asm-offsets.c 2017-04-18 17:54:21.000000000 +0200
  2037. @@ -36,6 +36,7 @@
  2038. BLANK();
  2039. OFFSET(TASK_TI_flags, task_struct, thread_info.flags);
  2040. + OFFSET(TASK_TI_preempt_lazy_count, task_struct, thread_info.preempt_lazy_count);
  2041. OFFSET(TASK_addr_limit, task_struct, thread.addr_limit);
  2042. BLANK();
  2043. @@ -91,4 +92,5 @@
  2044. BLANK();
  2045. DEFINE(PTREGS_SIZE, sizeof(struct pt_regs));
  2046. + DEFINE(_PREEMPT_ENABLED, PREEMPT_ENABLED);
  2047. }
  2048. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kernel/cpu/mcheck/mce.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kernel/cpu/mcheck/mce.c
  2049. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kernel/cpu/mcheck/mce.c 2017-04-16 10:37:37.000000000 +0200
  2050. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kernel/cpu/mcheck/mce.c 2017-04-18 17:54:21.000000000 +0200
  2051. @@ -41,6 +41,8 @@
  2052. #include <linux/debugfs.h>
  2053. #include <linux/irq_work.h>
  2054. #include <linux/export.h>
  2055. +#include <linux/jiffies.h>
  2056. +#include <linux/swork.h>
  2057. #include <linux/jump_label.h>
  2058. #include <asm/processor.h>
  2059. @@ -1317,7 +1319,7 @@
  2060. static unsigned long check_interval = INITIAL_CHECK_INTERVAL;
  2061. static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */
  2062. -static DEFINE_PER_CPU(struct timer_list, mce_timer);
  2063. +static DEFINE_PER_CPU(struct hrtimer, mce_timer);
  2064. static unsigned long mce_adjust_timer_default(unsigned long interval)
  2065. {
  2066. @@ -1326,32 +1328,18 @@
  2067. static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default;
  2068. -static void __restart_timer(struct timer_list *t, unsigned long interval)
  2069. +static enum hrtimer_restart __restart_timer(struct hrtimer *timer, unsigned long interval)
  2070. {
  2071. - unsigned long when = jiffies + interval;
  2072. - unsigned long flags;
  2073. -
  2074. - local_irq_save(flags);
  2075. -
  2076. - if (timer_pending(t)) {
  2077. - if (time_before(when, t->expires))
  2078. - mod_timer(t, when);
  2079. - } else {
  2080. - t->expires = round_jiffies(when);
  2081. - add_timer_on(t, smp_processor_id());
  2082. - }
  2083. -
  2084. - local_irq_restore(flags);
  2085. + if (!interval)
  2086. + return HRTIMER_NORESTART;
  2087. + hrtimer_forward_now(timer, ns_to_ktime(jiffies_to_nsecs(interval)));
  2088. + return HRTIMER_RESTART;
  2089. }
  2090. -static void mce_timer_fn(unsigned long data)
  2091. +static enum hrtimer_restart mce_timer_fn(struct hrtimer *timer)
  2092. {
  2093. - struct timer_list *t = this_cpu_ptr(&mce_timer);
  2094. - int cpu = smp_processor_id();
  2095. unsigned long iv;
  2096. - WARN_ON(cpu != data);
  2097. -
  2098. iv = __this_cpu_read(mce_next_interval);
  2099. if (mce_available(this_cpu_ptr(&cpu_info))) {
  2100. @@ -1374,7 +1362,7 @@
  2101. done:
  2102. __this_cpu_write(mce_next_interval, iv);
  2103. - __restart_timer(t, iv);
  2104. + return __restart_timer(timer, iv);
  2105. }
  2106. /*
  2107. @@ -1382,7 +1370,7 @@
  2108. */
  2109. void mce_timer_kick(unsigned long interval)
  2110. {
  2111. - struct timer_list *t = this_cpu_ptr(&mce_timer);
  2112. + struct hrtimer *t = this_cpu_ptr(&mce_timer);
  2113. unsigned long iv = __this_cpu_read(mce_next_interval);
  2114. __restart_timer(t, interval);
  2115. @@ -1397,7 +1385,7 @@
  2116. int cpu;
  2117. for_each_online_cpu(cpu)
  2118. - del_timer_sync(&per_cpu(mce_timer, cpu));
  2119. + hrtimer_cancel(&per_cpu(mce_timer, cpu));
  2120. }
  2121. static void mce_do_trigger(struct work_struct *work)
  2122. @@ -1407,6 +1395,56 @@
  2123. static DECLARE_WORK(mce_trigger_work, mce_do_trigger);
  2124. +static void __mce_notify_work(struct swork_event *event)
  2125. +{
  2126. + /* Not more than two messages every minute */
  2127. + static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);
  2128. +
  2129. + /* wake processes polling /dev/mcelog */
  2130. + wake_up_interruptible(&mce_chrdev_wait);
  2131. +
  2132. + /*
  2133. + * There is no risk of missing notifications because
  2134. + * work_pending is always cleared before the function is
  2135. + * executed.
  2136. + */
  2137. + if (mce_helper[0] && !work_pending(&mce_trigger_work))
  2138. + schedule_work(&mce_trigger_work);
  2139. +
  2140. + if (__ratelimit(&ratelimit))
  2141. + pr_info(HW_ERR "Machine check events logged\n");
  2142. +}
  2143. +
  2144. +#ifdef CONFIG_PREEMPT_RT_FULL
  2145. +static bool notify_work_ready __read_mostly;
  2146. +static struct swork_event notify_work;
  2147. +
  2148. +static int mce_notify_work_init(void)
  2149. +{
  2150. + int err;
  2151. +
  2152. + err = swork_get();
  2153. + if (err)
  2154. + return err;
  2155. +
  2156. + INIT_SWORK(&notify_work, __mce_notify_work);
  2157. + notify_work_ready = true;
  2158. + return 0;
  2159. +}
  2160. +
  2161. +static void mce_notify_work(void)
  2162. +{
  2163. + if (notify_work_ready)
  2164. + swork_queue(&notify_work);
  2165. +}
  2166. +#else
  2167. +static void mce_notify_work(void)
  2168. +{
  2169. + __mce_notify_work(NULL);
  2170. +}
  2171. +static inline int mce_notify_work_init(void) { return 0; }
  2172. +#endif
  2173. +
  2174. /*
  2175. * Notify the user(s) about new machine check events.
  2176. * Can be called from interrupt context, but not from machine check/NMI
  2177. @@ -1414,19 +1452,8 @@
  2178. */
  2179. int mce_notify_irq(void)
  2180. {
  2181. - /* Not more than two messages every minute */
  2182. - static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);
  2183. -
  2184. if (test_and_clear_bit(0, &mce_need_notify)) {
  2185. - /* wake processes polling /dev/mcelog */
  2186. - wake_up_interruptible(&mce_chrdev_wait);
  2187. -
  2188. - if (mce_helper[0])
  2189. - schedule_work(&mce_trigger_work);
  2190. -
  2191. - if (__ratelimit(&ratelimit))
  2192. - pr_info(HW_ERR "Machine check events logged\n");
  2193. -
  2194. + mce_notify_work();
  2195. return 1;
  2196. }
  2197. return 0;
  2198. @@ -1732,7 +1759,7 @@
  2199. }
  2200. }
  2201. -static void mce_start_timer(unsigned int cpu, struct timer_list *t)
  2202. +static void mce_start_timer(unsigned int cpu, struct hrtimer *t)
  2203. {
  2204. unsigned long iv = check_interval * HZ;
  2205. @@ -1741,16 +1768,17 @@
  2206. per_cpu(mce_next_interval, cpu) = iv;
  2207. - t->expires = round_jiffies(jiffies + iv);
  2208. - add_timer_on(t, cpu);
  2209. + hrtimer_start_range_ns(t, ns_to_ktime(jiffies_to_usecs(iv) * 1000ULL),
  2210. + 0, HRTIMER_MODE_REL_PINNED);
  2211. }
  2212. static void __mcheck_cpu_init_timer(void)
  2213. {
  2214. - struct timer_list *t = this_cpu_ptr(&mce_timer);
  2215. + struct hrtimer *t = this_cpu_ptr(&mce_timer);
  2216. unsigned int cpu = smp_processor_id();
  2217. - setup_pinned_timer(t, mce_timer_fn, cpu);
  2218. + hrtimer_init(t, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  2219. + t->function = mce_timer_fn;
  2220. mce_start_timer(cpu, t);
  2221. }
  2222. @@ -2475,6 +2503,8 @@
  2223. if (!mce_available(raw_cpu_ptr(&cpu_info)))
  2224. return;
  2225. + hrtimer_cancel(this_cpu_ptr(&mce_timer));
  2226. +
  2227. if (!(action & CPU_TASKS_FROZEN))
  2228. cmci_clear();
  2229. @@ -2497,6 +2527,7 @@
  2230. if (b->init)
  2231. wrmsrl(msr_ops.ctl(i), b->ctl);
  2232. }
  2233. + __mcheck_cpu_init_timer();
  2234. }
  2235. /* Get notified when a cpu comes on/off. Be hotplug friendly. */
  2236. @@ -2504,7 +2535,6 @@
  2237. mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
  2238. {
  2239. unsigned int cpu = (unsigned long)hcpu;
  2240. - struct timer_list *t = &per_cpu(mce_timer, cpu);
  2241. switch (action & ~CPU_TASKS_FROZEN) {
  2242. case CPU_ONLINE:
  2243. @@ -2524,11 +2554,9 @@
  2244. break;
  2245. case CPU_DOWN_PREPARE:
  2246. smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
  2247. - del_timer_sync(t);
  2248. break;
  2249. case CPU_DOWN_FAILED:
  2250. smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
  2251. - mce_start_timer(cpu, t);
  2252. break;
  2253. }
  2254. @@ -2567,6 +2595,10 @@
  2255. goto err_out;
  2256. }
  2257. + err = mce_notify_work_init();
  2258. + if (err)
  2259. + goto err_out;
  2260. +
  2261. if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) {
  2262. err = -ENOMEM;
  2263. goto err_out;
  2264. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kernel/irq_32.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kernel/irq_32.c
  2265. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kernel/irq_32.c 2017-04-16 10:37:37.000000000 +0200
  2266. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kernel/irq_32.c 2017-04-18 17:54:21.000000000 +0200
  2267. @@ -127,6 +127,7 @@
  2268. cpu, per_cpu(hardirq_stack, cpu), per_cpu(softirq_stack, cpu));
  2269. }
  2270. +#ifndef CONFIG_PREEMPT_RT_FULL
  2271. void do_softirq_own_stack(void)
  2272. {
  2273. struct irq_stack *irqstk;
  2274. @@ -143,6 +144,7 @@
  2275. call_on_stack(__do_softirq, isp);
  2276. }
  2277. +#endif
  2278. bool handle_irq(struct irq_desc *desc, struct pt_regs *regs)
  2279. {
  2280. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kernel/process_32.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kernel/process_32.c
  2281. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kernel/process_32.c 2017-04-16 10:37:37.000000000 +0200
  2282. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kernel/process_32.c 2017-04-18 17:54:21.000000000 +0200
  2283. @@ -35,6 +35,7 @@
  2284. #include <linux/uaccess.h>
  2285. #include <linux/io.h>
  2286. #include <linux/kdebug.h>
  2287. +#include <linux/highmem.h>
  2288. #include <asm/pgtable.h>
  2289. #include <asm/ldt.h>
  2290. @@ -195,6 +196,35 @@
  2291. }
  2292. EXPORT_SYMBOL_GPL(start_thread);
  2293. +#ifdef CONFIG_PREEMPT_RT_FULL
  2294. +static void switch_kmaps(struct task_struct *prev_p, struct task_struct *next_p)
  2295. +{
  2296. + int i;
  2297. +
  2298. + /*
  2299. + * Clear @prev's kmap_atomic mappings
  2300. + */
  2301. + for (i = 0; i < prev_p->kmap_idx; i++) {
  2302. + int idx = i + KM_TYPE_NR * smp_processor_id();
  2303. + pte_t *ptep = kmap_pte - idx;
  2304. +
  2305. + kpte_clear_flush(ptep, __fix_to_virt(FIX_KMAP_BEGIN + idx));
  2306. + }
  2307. + /*
  2308. + * Restore @next_p's kmap_atomic mappings
  2309. + */
  2310. + for (i = 0; i < next_p->kmap_idx; i++) {
  2311. + int idx = i + KM_TYPE_NR * smp_processor_id();
  2312. +
  2313. + if (!pte_none(next_p->kmap_pte[i]))
  2314. + set_pte(kmap_pte - idx, next_p->kmap_pte[i]);
  2315. + }
  2316. +}
  2317. +#else
  2318. +static inline void
  2319. +switch_kmaps(struct task_struct *prev_p, struct task_struct *next_p) { }
  2320. +#endif
  2321. +
  2322. /*
  2323. * switch_to(x,y) should switch tasks from x to y.
  2324. @@ -271,6 +301,8 @@
  2325. task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT))
  2326. __switch_to_xtra(prev_p, next_p, tss);
  2327. + switch_kmaps(prev_p, next_p);
  2328. +
  2329. /*
  2330. * Leave lazy mode, flushing any hypercalls made here.
  2331. * This must be done before restoring TLS segments so
  2332. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kvm/lapic.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kvm/lapic.c
  2333. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kvm/lapic.c 2017-04-16 10:37:37.000000000 +0200
  2334. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kvm/lapic.c 2017-04-18 17:54:21.000000000 +0200
  2335. @@ -1939,6 +1939,7 @@
  2336. hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
  2337. HRTIMER_MODE_ABS_PINNED);
  2338. apic->lapic_timer.timer.function = apic_timer_fn;
  2339. + apic->lapic_timer.timer.irqsafe = 1;
  2340. /*
  2341. * APIC is created enabled. This will prevent kvm_lapic_set_base from
  2342. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kvm/x86.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kvm/x86.c
  2343. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/kvm/x86.c 2017-04-16 10:37:37.000000000 +0200
  2344. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/kvm/x86.c 2017-04-18 17:54:21.000000000 +0200
  2345. @@ -5933,6 +5933,13 @@
  2346. goto out;
  2347. }
  2348. +#ifdef CONFIG_PREEMPT_RT_FULL
  2349. + if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
  2350. + printk(KERN_ERR "RT requires X86_FEATURE_CONSTANT_TSC\n");
  2351. + return -EOPNOTSUPP;
  2352. + }
  2353. +#endif
  2354. +
  2355. r = kvm_mmu_module_init();
  2356. if (r)
  2357. goto out_free_percpu;
  2358. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/mm/highmem_32.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/mm/highmem_32.c
  2359. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/mm/highmem_32.c 2017-04-16 10:37:37.000000000 +0200
  2360. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/mm/highmem_32.c 2017-04-18 17:54:21.000000000 +0200
  2361. @@ -32,10 +32,11 @@
  2362. */
  2363. void *kmap_atomic_prot(struct page *page, pgprot_t prot)
  2364. {
  2365. + pte_t pte = mk_pte(page, prot);
  2366. unsigned long vaddr;
  2367. int idx, type;
  2368. - preempt_disable();
  2369. + preempt_disable_nort();
  2370. pagefault_disable();
  2371. if (!PageHighMem(page))
  2372. @@ -45,7 +46,10 @@
  2373. idx = type + KM_TYPE_NR*smp_processor_id();
  2374. vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
  2375. BUG_ON(!pte_none(*(kmap_pte-idx)));
  2376. - set_pte(kmap_pte-idx, mk_pte(page, prot));
  2377. +#ifdef CONFIG_PREEMPT_RT_FULL
  2378. + current->kmap_pte[type] = pte;
  2379. +#endif
  2380. + set_pte(kmap_pte-idx, pte);
  2381. arch_flush_lazy_mmu_mode();
  2382. return (void *)vaddr;
  2383. @@ -88,6 +92,9 @@
  2384. * is a bad idea also, in case the page changes cacheability
  2385. * attributes or becomes a protected page in a hypervisor.
  2386. */
  2387. +#ifdef CONFIG_PREEMPT_RT_FULL
  2388. + current->kmap_pte[type] = __pte(0);
  2389. +#endif
  2390. kpte_clear_flush(kmap_pte-idx, vaddr);
  2391. kmap_atomic_idx_pop();
  2392. arch_flush_lazy_mmu_mode();
  2393. @@ -100,7 +107,7 @@
  2394. #endif
  2395. pagefault_enable();
  2396. - preempt_enable();
  2397. + preempt_enable_nort();
  2398. }
  2399. EXPORT_SYMBOL(__kunmap_atomic);
  2400. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/mm/iomap_32.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/mm/iomap_32.c
  2401. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/mm/iomap_32.c 2017-04-16 10:37:37.000000000 +0200
  2402. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/mm/iomap_32.c 2017-04-18 17:54:21.000000000 +0200
  2403. @@ -56,6 +56,7 @@
  2404. void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot)
  2405. {
  2406. + pte_t pte = pfn_pte(pfn, prot);
  2407. unsigned long vaddr;
  2408. int idx, type;
  2409. @@ -65,7 +66,12 @@
  2410. type = kmap_atomic_idx_push();
  2411. idx = type + KM_TYPE_NR * smp_processor_id();
  2412. vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
  2413. - set_pte(kmap_pte - idx, pfn_pte(pfn, prot));
  2414. + WARN_ON(!pte_none(*(kmap_pte - idx)));
  2415. +
  2416. +#ifdef CONFIG_PREEMPT_RT_FULL
  2417. + current->kmap_pte[type] = pte;
  2418. +#endif
  2419. + set_pte(kmap_pte - idx, pte);
  2420. arch_flush_lazy_mmu_mode();
  2421. return (void *)vaddr;
  2422. @@ -113,6 +119,9 @@
  2423. * is a bad idea also, in case the page changes cacheability
  2424. * attributes or becomes a protected page in a hypervisor.
  2425. */
  2426. +#ifdef CONFIG_PREEMPT_RT_FULL
  2427. + current->kmap_pte[type] = __pte(0);
  2428. +#endif
  2429. kpte_clear_flush(kmap_pte-idx, vaddr);
  2430. kmap_atomic_idx_pop();
  2431. }
  2432. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/mm/pageattr.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/mm/pageattr.c
  2433. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/mm/pageattr.c 2017-04-16 10:37:37.000000000 +0200
  2434. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/mm/pageattr.c 2017-04-18 17:54:21.000000000 +0200
  2435. @@ -214,7 +214,15 @@
  2436. int in_flags, struct page **pages)
  2437. {
  2438. unsigned int i, level;
  2439. +#ifdef CONFIG_PREEMPT
  2440. + /*
  2441. + * Avoid wbinvd() because it causes latencies on all CPUs,
  2442. + * regardless of any CPU isolation that may be in effect.
  2443. + */
  2444. + unsigned long do_wbinvd = 0;
  2445. +#else
  2446. unsigned long do_wbinvd = cache && numpages >= 1024; /* 4M threshold */
  2447. +#endif
  2448. BUG_ON(irqs_disabled());
  2449. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/platform/uv/tlb_uv.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/platform/uv/tlb_uv.c
  2450. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/platform/uv/tlb_uv.c 2017-04-16 10:37:37.000000000 +0200
  2451. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/platform/uv/tlb_uv.c 2017-04-18 17:54:21.000000000 +0200
  2452. @@ -748,9 +748,9 @@
  2453. quiesce_local_uvhub(hmaster);
  2454. - spin_lock(&hmaster->queue_lock);
  2455. + raw_spin_lock(&hmaster->queue_lock);
  2456. reset_with_ipi(&bau_desc->distribution, bcp);
  2457. - spin_unlock(&hmaster->queue_lock);
  2458. + raw_spin_unlock(&hmaster->queue_lock);
  2459. end_uvhub_quiesce(hmaster);
  2460. @@ -770,9 +770,9 @@
  2461. quiesce_local_uvhub(hmaster);
  2462. - spin_lock(&hmaster->queue_lock);
  2463. + raw_spin_lock(&hmaster->queue_lock);
  2464. reset_with_ipi(&bau_desc->distribution, bcp);
  2465. - spin_unlock(&hmaster->queue_lock);
  2466. + raw_spin_unlock(&hmaster->queue_lock);
  2467. end_uvhub_quiesce(hmaster);
  2468. @@ -793,7 +793,7 @@
  2469. cycles_t tm1;
  2470. hmaster = bcp->uvhub_master;
  2471. - spin_lock(&hmaster->disable_lock);
  2472. + raw_spin_lock(&hmaster->disable_lock);
  2473. if (!bcp->baudisabled) {
  2474. stat->s_bau_disabled++;
  2475. tm1 = get_cycles();
  2476. @@ -806,7 +806,7 @@
  2477. }
  2478. }
  2479. }
  2480. - spin_unlock(&hmaster->disable_lock);
  2481. + raw_spin_unlock(&hmaster->disable_lock);
  2482. }
  2483. static void count_max_concurr(int stat, struct bau_control *bcp,
  2484. @@ -869,7 +869,7 @@
  2485. */
  2486. static void uv1_throttle(struct bau_control *hmaster, struct ptc_stats *stat)
  2487. {
  2488. - spinlock_t *lock = &hmaster->uvhub_lock;
  2489. + raw_spinlock_t *lock = &hmaster->uvhub_lock;
  2490. atomic_t *v;
  2491. v = &hmaster->active_descriptor_count;
  2492. @@ -1002,7 +1002,7 @@
  2493. struct bau_control *hmaster;
  2494. hmaster = bcp->uvhub_master;
  2495. - spin_lock(&hmaster->disable_lock);
  2496. + raw_spin_lock(&hmaster->disable_lock);
  2497. if (bcp->baudisabled && (get_cycles() >= bcp->set_bau_on_time)) {
  2498. stat->s_bau_reenabled++;
  2499. for_each_present_cpu(tcpu) {
  2500. @@ -1014,10 +1014,10 @@
  2501. tbcp->period_giveups = 0;
  2502. }
  2503. }
  2504. - spin_unlock(&hmaster->disable_lock);
  2505. + raw_spin_unlock(&hmaster->disable_lock);
  2506. return 0;
  2507. }
  2508. - spin_unlock(&hmaster->disable_lock);
  2509. + raw_spin_unlock(&hmaster->disable_lock);
  2510. return -1;
  2511. }
  2512. @@ -1940,9 +1940,9 @@
  2513. bcp->cong_reps = congested_reps;
  2514. bcp->disabled_period = sec_2_cycles(disabled_period);
  2515. bcp->giveup_limit = giveup_limit;
  2516. - spin_lock_init(&bcp->queue_lock);
  2517. - spin_lock_init(&bcp->uvhub_lock);
  2518. - spin_lock_init(&bcp->disable_lock);
  2519. + raw_spin_lock_init(&bcp->queue_lock);
  2520. + raw_spin_lock_init(&bcp->uvhub_lock);
  2521. + raw_spin_lock_init(&bcp->disable_lock);
  2522. }
  2523. }
  2524. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/platform/uv/uv_time.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/platform/uv/uv_time.c
  2525. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/arch/x86/platform/uv/uv_time.c 2017-04-16 10:37:37.000000000 +0200
  2526. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/arch/x86/platform/uv/uv_time.c 2017-04-18 17:54:21.000000000 +0200
  2527. @@ -57,7 +57,7 @@
  2528. /* There is one of these allocated per node */
  2529. struct uv_rtc_timer_head {
  2530. - spinlock_t lock;
  2531. + raw_spinlock_t lock;
  2532. /* next cpu waiting for timer, local node relative: */
  2533. int next_cpu;
  2534. /* number of cpus on this node: */
  2535. @@ -177,7 +177,7 @@
  2536. uv_rtc_deallocate_timers();
  2537. return -ENOMEM;
  2538. }
  2539. - spin_lock_init(&head->lock);
  2540. + raw_spin_lock_init(&head->lock);
  2541. head->ncpus = uv_blade_nr_possible_cpus(bid);
  2542. head->next_cpu = -1;
  2543. blade_info[bid] = head;
  2544. @@ -231,7 +231,7 @@
  2545. unsigned long flags;
  2546. int next_cpu;
  2547. - spin_lock_irqsave(&head->lock, flags);
  2548. + raw_spin_lock_irqsave(&head->lock, flags);
  2549. next_cpu = head->next_cpu;
  2550. *t = expires;
  2551. @@ -243,12 +243,12 @@
  2552. if (uv_setup_intr(cpu, expires)) {
  2553. *t = ULLONG_MAX;
  2554. uv_rtc_find_next_timer(head, pnode);
  2555. - spin_unlock_irqrestore(&head->lock, flags);
  2556. + raw_spin_unlock_irqrestore(&head->lock, flags);
  2557. return -ETIME;
  2558. }
  2559. }
  2560. - spin_unlock_irqrestore(&head->lock, flags);
  2561. + raw_spin_unlock_irqrestore(&head->lock, flags);
  2562. return 0;
  2563. }
  2564. @@ -267,7 +267,7 @@
  2565. unsigned long flags;
  2566. int rc = 0;
  2567. - spin_lock_irqsave(&head->lock, flags);
  2568. + raw_spin_lock_irqsave(&head->lock, flags);
  2569. if ((head->next_cpu == bcpu && uv_read_rtc(NULL) >= *t) || force)
  2570. rc = 1;
  2571. @@ -279,7 +279,7 @@
  2572. uv_rtc_find_next_timer(head, pnode);
  2573. }
  2574. - spin_unlock_irqrestore(&head->lock, flags);
  2575. + raw_spin_unlock_irqrestore(&head->lock, flags);
  2576. return rc;
  2577. }
  2578. @@ -299,13 +299,18 @@
  2579. static cycle_t uv_read_rtc(struct clocksource *cs)
  2580. {
  2581. unsigned long offset;
  2582. + cycle_t cycles;
  2583. + preempt_disable();
  2584. if (uv_get_min_hub_revision_id() == 1)
  2585. offset = 0;
  2586. else
  2587. offset = (uv_blade_processor_id() * L1_CACHE_BYTES) % PAGE_SIZE;
  2588. - return (cycle_t)uv_read_local_mmr(UVH_RTC | offset);
  2589. + cycles = (cycle_t)uv_read_local_mmr(UVH_RTC | offset);
  2590. + preempt_enable();
  2591. +
  2592. + return cycles;
  2593. }
  2594. /*
  2595. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/block/blk-core.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/block/blk-core.c
  2596. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/block/blk-core.c 2017-04-16 10:37:38.000000000 +0200
  2597. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/block/blk-core.c 2017-04-18 17:54:21.000000000 +0200
  2598. @@ -125,6 +125,9 @@
  2599. INIT_LIST_HEAD(&rq->queuelist);
  2600. INIT_LIST_HEAD(&rq->timeout_list);
  2601. +#ifdef CONFIG_PREEMPT_RT_FULL
  2602. + INIT_WORK(&rq->work, __blk_mq_complete_request_remote_work);
  2603. +#endif
  2604. rq->cpu = -1;
  2605. rq->q = q;
  2606. rq->__sector = (sector_t) -1;
  2607. @@ -233,7 +236,7 @@
  2608. **/
  2609. void blk_start_queue(struct request_queue *q)
  2610. {
  2611. - WARN_ON(!irqs_disabled());
  2612. + WARN_ON_NONRT(!irqs_disabled());
  2613. queue_flag_clear(QUEUE_FLAG_STOPPED, q);
  2614. __blk_run_queue(q);
  2615. @@ -659,7 +662,7 @@
  2616. if (nowait)
  2617. return -EBUSY;
  2618. - ret = wait_event_interruptible(q->mq_freeze_wq,
  2619. + ret = swait_event_interruptible(q->mq_freeze_wq,
  2620. !atomic_read(&q->mq_freeze_depth) ||
  2621. blk_queue_dying(q));
  2622. if (blk_queue_dying(q))
  2623. @@ -679,7 +682,7 @@
  2624. struct request_queue *q =
  2625. container_of(ref, struct request_queue, q_usage_counter);
  2626. - wake_up_all(&q->mq_freeze_wq);
  2627. + swake_up_all(&q->mq_freeze_wq);
  2628. }
  2629. static void blk_rq_timed_out_timer(unsigned long data)
  2630. @@ -748,7 +751,7 @@
  2631. q->bypass_depth = 1;
  2632. __set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags);
  2633. - init_waitqueue_head(&q->mq_freeze_wq);
  2634. + init_swait_queue_head(&q->mq_freeze_wq);
  2635. /*
  2636. * Init percpu_ref in atomic mode so that it's faster to shutdown.
  2637. @@ -3200,7 +3203,7 @@
  2638. blk_run_queue_async(q);
  2639. else
  2640. __blk_run_queue(q);
  2641. - spin_unlock(q->queue_lock);
  2642. + spin_unlock_irq(q->queue_lock);
  2643. }
  2644. static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
  2645. @@ -3248,7 +3251,6 @@
  2646. void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
  2647. {
  2648. struct request_queue *q;
  2649. - unsigned long flags;
  2650. struct request *rq;
  2651. LIST_HEAD(list);
  2652. unsigned int depth;
  2653. @@ -3268,11 +3270,6 @@
  2654. q = NULL;
  2655. depth = 0;
  2656. - /*
  2657. - * Save and disable interrupts here, to avoid doing it for every
  2658. - * queue lock we have to take.
  2659. - */
  2660. - local_irq_save(flags);
  2661. while (!list_empty(&list)) {
  2662. rq = list_entry_rq(list.next);
  2663. list_del_init(&rq->queuelist);
  2664. @@ -3285,7 +3282,7 @@
  2665. queue_unplugged(q, depth, from_schedule);
  2666. q = rq->q;
  2667. depth = 0;
  2668. - spin_lock(q->queue_lock);
  2669. + spin_lock_irq(q->queue_lock);
  2670. }
  2671. /*
  2672. @@ -3312,8 +3309,6 @@
  2673. */
  2674. if (q)
  2675. queue_unplugged(q, depth, from_schedule);
  2676. -
  2677. - local_irq_restore(flags);
  2678. }
  2679. void blk_finish_plug(struct blk_plug *plug)
  2680. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/block/blk-ioc.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/block/blk-ioc.c
  2681. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/block/blk-ioc.c 2017-04-16 10:37:38.000000000 +0200
  2682. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/block/blk-ioc.c 2017-04-18 17:54:21.000000000 +0200
  2683. @@ -7,6 +7,7 @@
  2684. #include <linux/bio.h>
  2685. #include <linux/blkdev.h>
  2686. #include <linux/slab.h>
  2687. +#include <linux/delay.h>
  2688. #include "blk.h"
  2689. @@ -109,7 +110,7 @@
  2690. spin_unlock(q->queue_lock);
  2691. } else {
  2692. spin_unlock_irqrestore(&ioc->lock, flags);
  2693. - cpu_relax();
  2694. + cpu_chill();
  2695. spin_lock_irqsave_nested(&ioc->lock, flags, 1);
  2696. }
  2697. }
  2698. @@ -187,7 +188,7 @@
  2699. spin_unlock(icq->q->queue_lock);
  2700. } else {
  2701. spin_unlock_irqrestore(&ioc->lock, flags);
  2702. - cpu_relax();
  2703. + cpu_chill();
  2704. goto retry;
  2705. }
  2706. }
  2707. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/block/blk-mq.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/block/blk-mq.c
  2708. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/block/blk-mq.c 2017-04-16 10:37:38.000000000 +0200
  2709. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/block/blk-mq.c 2017-04-18 17:54:21.000000000 +0200
  2710. @@ -72,7 +72,7 @@
  2711. static void blk_mq_freeze_queue_wait(struct request_queue *q)
  2712. {
  2713. - wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
  2714. + swait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
  2715. }
  2716. /*
  2717. @@ -110,7 +110,7 @@
  2718. WARN_ON_ONCE(freeze_depth < 0);
  2719. if (!freeze_depth) {
  2720. percpu_ref_reinit(&q->q_usage_counter);
  2721. - wake_up_all(&q->mq_freeze_wq);
  2722. + swake_up_all(&q->mq_freeze_wq);
  2723. }
  2724. }
  2725. EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
  2726. @@ -129,7 +129,7 @@
  2727. * dying, we need to ensure that processes currently waiting on
  2728. * the queue are notified as well.
  2729. */
  2730. - wake_up_all(&q->mq_freeze_wq);
  2731. + swake_up_all(&q->mq_freeze_wq);
  2732. }
  2733. bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
  2734. @@ -177,6 +177,9 @@
  2735. rq->resid_len = 0;
  2736. rq->sense = NULL;
  2737. +#ifdef CONFIG_PREEMPT_RT_FULL
  2738. + INIT_WORK(&rq->work, __blk_mq_complete_request_remote_work);
  2739. +#endif
  2740. INIT_LIST_HEAD(&rq->timeout_list);
  2741. rq->timeout = 0;
  2742. @@ -345,6 +348,17 @@
  2743. }
  2744. EXPORT_SYMBOL(blk_mq_end_request);
  2745. +#ifdef CONFIG_PREEMPT_RT_FULL
  2746. +
  2747. +void __blk_mq_complete_request_remote_work(struct work_struct *work)
  2748. +{
  2749. + struct request *rq = container_of(work, struct request, work);
  2750. +
  2751. + rq->q->softirq_done_fn(rq);
  2752. +}
  2753. +
  2754. +#else
  2755. +
  2756. static void __blk_mq_complete_request_remote(void *data)
  2757. {
  2758. struct request *rq = data;
  2759. @@ -352,6 +366,8 @@
  2760. rq->q->softirq_done_fn(rq);
  2761. }
  2762. +#endif
  2763. +
  2764. static void blk_mq_ipi_complete_request(struct request *rq)
  2765. {
  2766. struct blk_mq_ctx *ctx = rq->mq_ctx;
  2767. @@ -363,19 +379,23 @@
  2768. return;
  2769. }
  2770. - cpu = get_cpu();
  2771. + cpu = get_cpu_light();
  2772. if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
  2773. shared = cpus_share_cache(cpu, ctx->cpu);
  2774. if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
  2775. +#ifdef CONFIG_PREEMPT_RT_FULL
  2776. + schedule_work_on(ctx->cpu, &rq->work);
  2777. +#else
  2778. rq->csd.func = __blk_mq_complete_request_remote;
  2779. rq->csd.info = rq;
  2780. rq->csd.flags = 0;
  2781. smp_call_function_single_async(ctx->cpu, &rq->csd);
  2782. +#endif
  2783. } else {
  2784. rq->q->softirq_done_fn(rq);
  2785. }
  2786. - put_cpu();
  2787. + put_cpu_light();
  2788. }
  2789. static void __blk_mq_complete_request(struct request *rq)
  2790. @@ -906,14 +926,14 @@
  2791. return;
  2792. if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
  2793. - int cpu = get_cpu();
  2794. + int cpu = get_cpu_light();
  2795. if (cpumask_test_cpu(cpu, hctx->cpumask)) {
  2796. __blk_mq_run_hw_queue(hctx);
  2797. - put_cpu();
  2798. + put_cpu_light();
  2799. return;
  2800. }
  2801. - put_cpu();
  2802. + put_cpu_light();
  2803. }
  2804. kblockd_schedule_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work);
  2805. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/block/blk-mq.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/block/blk-mq.h
  2806. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/block/blk-mq.h 2017-04-16 10:37:38.000000000 +0200
  2807. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/block/blk-mq.h 2017-04-18 17:54:21.000000000 +0200
  2808. @@ -72,12 +72,12 @@
  2809. */
  2810. static inline struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q)
  2811. {
  2812. - return __blk_mq_get_ctx(q, get_cpu());
  2813. + return __blk_mq_get_ctx(q, get_cpu_light());
  2814. }
  2815. static inline void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
  2816. {
  2817. - put_cpu();
  2818. + put_cpu_light();
  2819. }
  2820. struct blk_mq_alloc_data {
  2821. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/block/blk-softirq.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/block/blk-softirq.c
  2822. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/block/blk-softirq.c 2017-04-16 10:37:38.000000000 +0200
  2823. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/block/blk-softirq.c 2017-04-18 17:54:21.000000000 +0200
  2824. @@ -51,6 +51,7 @@
  2825. raise_softirq_irqoff(BLOCK_SOFTIRQ);
  2826. local_irq_restore(flags);
  2827. + preempt_check_resched_rt();
  2828. }
  2829. /*
  2830. @@ -89,6 +90,7 @@
  2831. this_cpu_ptr(&blk_cpu_done));
  2832. raise_softirq_irqoff(BLOCK_SOFTIRQ);
  2833. local_irq_enable();
  2834. + preempt_check_resched_rt();
  2835. return 0;
  2836. }
  2837. @@ -141,6 +143,7 @@
  2838. goto do_local;
  2839. local_irq_restore(flags);
  2840. + preempt_check_resched_rt();
  2841. }
  2842. /**
  2843. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/block/bounce.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/block/bounce.c
  2844. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/block/bounce.c 2017-04-16 10:37:38.000000000 +0200
  2845. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/block/bounce.c 2017-04-18 17:54:21.000000000 +0200
  2846. @@ -55,11 +55,11 @@
  2847. unsigned long flags;
  2848. unsigned char *vto;
  2849. - local_irq_save(flags);
  2850. + local_irq_save_nort(flags);
  2851. vto = kmap_atomic(to->bv_page);
  2852. memcpy(vto + to->bv_offset, vfrom, to->bv_len);
  2853. kunmap_atomic(vto);
  2854. - local_irq_restore(flags);
  2855. + local_irq_restore_nort(flags);
  2856. }
  2857. #else /* CONFIG_HIGHMEM */
  2858. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/crypto/algapi.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/crypto/algapi.c
  2859. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/crypto/algapi.c 2017-04-16 10:37:38.000000000 +0200
  2860. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/crypto/algapi.c 2017-04-18 17:54:21.000000000 +0200
  2861. @@ -719,13 +719,13 @@
  2862. int crypto_register_notifier(struct notifier_block *nb)
  2863. {
  2864. - return blocking_notifier_chain_register(&crypto_chain, nb);
  2865. + return srcu_notifier_chain_register(&crypto_chain, nb);
  2866. }
  2867. EXPORT_SYMBOL_GPL(crypto_register_notifier);
  2868. int crypto_unregister_notifier(struct notifier_block *nb)
  2869. {
  2870. - return blocking_notifier_chain_unregister(&crypto_chain, nb);
  2871. + return srcu_notifier_chain_unregister(&crypto_chain, nb);
  2872. }
  2873. EXPORT_SYMBOL_GPL(crypto_unregister_notifier);
  2874. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/crypto/api.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/crypto/api.c
  2875. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/crypto/api.c 2017-04-16 10:37:38.000000000 +0200
  2876. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/crypto/api.c 2017-04-18 17:54:21.000000000 +0200
  2877. @@ -31,7 +31,7 @@
  2878. DECLARE_RWSEM(crypto_alg_sem);
  2879. EXPORT_SYMBOL_GPL(crypto_alg_sem);
  2880. -BLOCKING_NOTIFIER_HEAD(crypto_chain);
  2881. +SRCU_NOTIFIER_HEAD(crypto_chain);
  2882. EXPORT_SYMBOL_GPL(crypto_chain);
  2883. static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg);
  2884. @@ -236,10 +236,10 @@
  2885. {
  2886. int ok;
  2887. - ok = blocking_notifier_call_chain(&crypto_chain, val, v);
  2888. + ok = srcu_notifier_call_chain(&crypto_chain, val, v);
  2889. if (ok == NOTIFY_DONE) {
  2890. request_module("cryptomgr");
  2891. - ok = blocking_notifier_call_chain(&crypto_chain, val, v);
  2892. + ok = srcu_notifier_call_chain(&crypto_chain, val, v);
  2893. }
  2894. return ok;
  2895. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/crypto/internal.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/crypto/internal.h
  2896. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/crypto/internal.h 2017-04-16 10:37:38.000000000 +0200
  2897. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/crypto/internal.h 2017-04-18 17:54:21.000000000 +0200
  2898. @@ -47,7 +47,7 @@
  2899. extern struct list_head crypto_alg_list;
  2900. extern struct rw_semaphore crypto_alg_sem;
  2901. -extern struct blocking_notifier_head crypto_chain;
  2902. +extern struct srcu_notifier_head crypto_chain;
  2903. #ifdef CONFIG_PROC_FS
  2904. void __init crypto_init_proc(void);
  2905. @@ -146,7 +146,7 @@
  2906. static inline void crypto_notify(unsigned long val, void *v)
  2907. {
  2908. - blocking_notifier_call_chain(&crypto_chain, val, v);
  2909. + srcu_notifier_call_chain(&crypto_chain, val, v);
  2910. }
  2911. #endif /* _CRYPTO_INTERNAL_H */
  2912. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/acpi/acpica/acglobal.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/acpi/acpica/acglobal.h
  2913. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/acpi/acpica/acglobal.h 2017-04-16 10:37:38.000000000 +0200
  2914. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/acpi/acpica/acglobal.h 2017-04-18 17:54:22.000000000 +0200
  2915. @@ -116,7 +116,7 @@
  2916. * interrupt level
  2917. */
  2918. ACPI_GLOBAL(acpi_spinlock, acpi_gbl_gpe_lock); /* For GPE data structs and registers */
  2919. -ACPI_GLOBAL(acpi_spinlock, acpi_gbl_hardware_lock); /* For ACPI H/W except GPE registers */
  2920. +ACPI_GLOBAL(acpi_raw_spinlock, acpi_gbl_hardware_lock); /* For ACPI H/W except GPE registers */
  2921. ACPI_GLOBAL(acpi_spinlock, acpi_gbl_reference_count_lock);
  2922. /* Mutex for _OSI support */
  2923. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/acpi/acpica/hwregs.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/acpi/acpica/hwregs.c
  2924. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/acpi/acpica/hwregs.c 2017-04-16 10:37:38.000000000 +0200
  2925. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/acpi/acpica/hwregs.c 2017-04-18 17:54:22.000000000 +0200
  2926. @@ -363,14 +363,14 @@
  2927. ACPI_BITMASK_ALL_FIXED_STATUS,
  2928. ACPI_FORMAT_UINT64(acpi_gbl_xpm1a_status.address)));
  2929. - lock_flags = acpi_os_acquire_lock(acpi_gbl_hardware_lock);
  2930. + raw_spin_lock_irqsave(acpi_gbl_hardware_lock, lock_flags);
  2931. /* Clear the fixed events in PM1 A/B */
  2932. status = acpi_hw_register_write(ACPI_REGISTER_PM1_STATUS,
  2933. ACPI_BITMASK_ALL_FIXED_STATUS);
  2934. - acpi_os_release_lock(acpi_gbl_hardware_lock, lock_flags);
  2935. + raw_spin_unlock_irqrestore(acpi_gbl_hardware_lock, lock_flags);
  2936. if (ACPI_FAILURE(status)) {
  2937. goto exit;
  2938. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/acpi/acpica/hwxface.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/acpi/acpica/hwxface.c
  2939. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/acpi/acpica/hwxface.c 2017-04-16 10:37:38.000000000 +0200
  2940. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/acpi/acpica/hwxface.c 2017-04-18 17:54:22.000000000 +0200
  2941. @@ -373,7 +373,7 @@
  2942. return_ACPI_STATUS(AE_BAD_PARAMETER);
  2943. }
  2944. - lock_flags = acpi_os_acquire_lock(acpi_gbl_hardware_lock);
  2945. + raw_spin_lock_irqsave(acpi_gbl_hardware_lock, lock_flags);
  2946. /*
  2947. * At this point, we know that the parent register is one of the
  2948. @@ -434,7 +434,7 @@
  2949. unlock_and_exit:
  2950. - acpi_os_release_lock(acpi_gbl_hardware_lock, lock_flags);
  2951. + raw_spin_unlock_irqrestore(acpi_gbl_hardware_lock, lock_flags);
  2952. return_ACPI_STATUS(status);
  2953. }
  2954. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/acpi/acpica/utmutex.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/acpi/acpica/utmutex.c
  2955. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/acpi/acpica/utmutex.c 2017-04-16 10:37:39.000000000 +0200
  2956. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/acpi/acpica/utmutex.c 2017-04-18 17:54:22.000000000 +0200
  2957. @@ -88,7 +88,7 @@
  2958. return_ACPI_STATUS (status);
  2959. }
  2960. - status = acpi_os_create_lock (&acpi_gbl_hardware_lock);
  2961. + status = acpi_os_create_raw_lock (&acpi_gbl_hardware_lock);
  2962. if (ACPI_FAILURE (status)) {
  2963. return_ACPI_STATUS (status);
  2964. }
  2965. @@ -145,7 +145,7 @@
  2966. /* Delete the spinlocks */
  2967. acpi_os_delete_lock(acpi_gbl_gpe_lock);
  2968. - acpi_os_delete_lock(acpi_gbl_hardware_lock);
  2969. + acpi_os_delete_raw_lock(acpi_gbl_hardware_lock);
  2970. acpi_os_delete_lock(acpi_gbl_reference_count_lock);
  2971. /* Delete the reader/writer lock */
  2972. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ata/libata-sff.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ata/libata-sff.c
  2973. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ata/libata-sff.c 2017-04-16 10:37:39.000000000 +0200
  2974. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ata/libata-sff.c 2017-04-18 17:54:22.000000000 +0200
  2975. @@ -678,9 +678,9 @@
  2976. unsigned long flags;
  2977. unsigned int consumed;
  2978. - local_irq_save(flags);
  2979. + local_irq_save_nort(flags);
  2980. consumed = ata_sff_data_xfer32(dev, buf, buflen, rw);
  2981. - local_irq_restore(flags);
  2982. + local_irq_restore_nort(flags);
  2983. return consumed;
  2984. }
  2985. @@ -719,7 +719,7 @@
  2986. unsigned long flags;
  2987. /* FIXME: use a bounce buffer */
  2988. - local_irq_save(flags);
  2989. + local_irq_save_nort(flags);
  2990. buf = kmap_atomic(page);
  2991. /* do the actual data transfer */
  2992. @@ -727,7 +727,7 @@
  2993. do_write);
  2994. kunmap_atomic(buf);
  2995. - local_irq_restore(flags);
  2996. + local_irq_restore_nort(flags);
  2997. } else {
  2998. buf = page_address(page);
  2999. ap->ops->sff_data_xfer(qc->dev, buf + offset, qc->sect_size,
  3000. @@ -864,7 +864,7 @@
  3001. unsigned long flags;
  3002. /* FIXME: use bounce buffer */
  3003. - local_irq_save(flags);
  3004. + local_irq_save_nort(flags);
  3005. buf = kmap_atomic(page);
  3006. /* do the actual data transfer */
  3007. @@ -872,7 +872,7 @@
  3008. count, rw);
  3009. kunmap_atomic(buf);
  3010. - local_irq_restore(flags);
  3011. + local_irq_restore_nort(flags);
  3012. } else {
  3013. buf = page_address(page);
  3014. consumed = ap->ops->sff_data_xfer(dev, buf + offset,
  3015. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/block/zram/zcomp.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/block/zram/zcomp.c
  3016. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/block/zram/zcomp.c 2017-04-16 10:37:39.000000000 +0200
  3017. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/block/zram/zcomp.c 2017-04-18 17:54:22.000000000 +0200
  3018. @@ -118,12 +118,19 @@
  3019. struct zcomp_strm *zcomp_stream_get(struct zcomp *comp)
  3020. {
  3021. - return *get_cpu_ptr(comp->stream);
  3022. + struct zcomp_strm *zstrm;
  3023. +
  3024. + zstrm = *this_cpu_ptr(comp->stream);
  3025. + spin_lock(&zstrm->zcomp_lock);
  3026. + return zstrm;
  3027. }
  3028. void zcomp_stream_put(struct zcomp *comp)
  3029. {
  3030. - put_cpu_ptr(comp->stream);
  3031. + struct zcomp_strm *zstrm;
  3032. +
  3033. + zstrm = *this_cpu_ptr(comp->stream);
  3034. + spin_unlock(&zstrm->zcomp_lock);
  3035. }
  3036. int zcomp_compress(struct zcomp_strm *zstrm,
  3037. @@ -174,6 +181,7 @@
  3038. pr_err("Can't allocate a compression stream\n");
  3039. return NOTIFY_BAD;
  3040. }
  3041. + spin_lock_init(&zstrm->zcomp_lock);
  3042. *per_cpu_ptr(comp->stream, cpu) = zstrm;
  3043. break;
  3044. case CPU_DEAD:
  3045. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/block/zram/zcomp.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/block/zram/zcomp.h
  3046. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/block/zram/zcomp.h 2017-04-16 10:37:39.000000000 +0200
  3047. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/block/zram/zcomp.h 2017-04-18 17:54:22.000000000 +0200
  3048. @@ -14,6 +14,7 @@
  3049. /* compression/decompression buffer */
  3050. void *buffer;
  3051. struct crypto_comp *tfm;
  3052. + spinlock_t zcomp_lock;
  3053. };
  3054. /* dynamic per-device compression frontend */
  3055. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/block/zram/zram_drv.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/block/zram/zram_drv.c
  3056. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/block/zram/zram_drv.c 2017-04-16 10:37:39.000000000 +0200
  3057. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/block/zram/zram_drv.c 2017-04-18 17:54:22.000000000 +0200
  3058. @@ -528,6 +528,8 @@
  3059. goto out_error;
  3060. }
  3061. + zram_meta_init_table_locks(meta, disksize);
  3062. +
  3063. return meta;
  3064. out_error:
  3065. @@ -575,28 +577,28 @@
  3066. struct zram_meta *meta = zram->meta;
  3067. unsigned long handle;
  3068. unsigned int size;
  3069. + struct zcomp_strm *zstrm;
  3070. - bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
  3071. + zram_lock_table(&meta->table[index]);
  3072. handle = meta->table[index].handle;
  3073. size = zram_get_obj_size(meta, index);
  3074. if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
  3075. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3076. + zram_unlock_table(&meta->table[index]);
  3077. clear_page(mem);
  3078. return 0;
  3079. }
  3080. + zstrm = zcomp_stream_get(zram->comp);
  3081. cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
  3082. if (size == PAGE_SIZE) {
  3083. copy_page(mem, cmem);
  3084. } else {
  3085. - struct zcomp_strm *zstrm = zcomp_stream_get(zram->comp);
  3086. -
  3087. ret = zcomp_decompress(zstrm, cmem, size, mem);
  3088. - zcomp_stream_put(zram->comp);
  3089. }
  3090. zs_unmap_object(meta->mem_pool, handle);
  3091. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3092. + zcomp_stream_put(zram->comp);
  3093. + zram_unlock_table(&meta->table[index]);
  3094. /* Should NEVER happen. Return bio error if it does. */
  3095. if (unlikely(ret)) {
  3096. @@ -616,14 +618,14 @@
  3097. struct zram_meta *meta = zram->meta;
  3098. page = bvec->bv_page;
  3099. - bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
  3100. + zram_lock_table(&meta->table[index]);
  3101. if (unlikely(!meta->table[index].handle) ||
  3102. zram_test_flag(meta, index, ZRAM_ZERO)) {
  3103. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3104. + zram_unlock_table(&meta->table[index]);
  3105. handle_zero_page(bvec);
  3106. return 0;
  3107. }
  3108. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3109. + zram_unlock_table(&meta->table[index]);
  3110. if (is_partial_io(bvec))
  3111. /* Use a temporary buffer to decompress the page */
  3112. @@ -700,10 +702,10 @@
  3113. if (user_mem)
  3114. kunmap_atomic(user_mem);
  3115. /* Free memory associated with this sector now. */
  3116. - bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
  3117. + zram_lock_table(&meta->table[index]);
  3118. zram_free_page(zram, index);
  3119. zram_set_flag(meta, index, ZRAM_ZERO);
  3120. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3121. + zram_unlock_table(&meta->table[index]);
  3122. atomic64_inc(&zram->stats.zero_pages);
  3123. ret = 0;
  3124. @@ -794,12 +796,12 @@
  3125. * Free memory associated with this sector
  3126. * before overwriting unused sectors.
  3127. */
  3128. - bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
  3129. + zram_lock_table(&meta->table[index]);
  3130. zram_free_page(zram, index);
  3131. meta->table[index].handle = handle;
  3132. zram_set_obj_size(meta, index, clen);
  3133. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3134. + zram_unlock_table(&meta->table[index]);
  3135. /* Update stats */
  3136. atomic64_add(clen, &zram->stats.compr_data_size);
  3137. @@ -842,9 +844,9 @@
  3138. }
  3139. while (n >= PAGE_SIZE) {
  3140. - bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
  3141. + zram_lock_table(&meta->table[index]);
  3142. zram_free_page(zram, index);
  3143. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3144. + zram_unlock_table(&meta->table[index]);
  3145. atomic64_inc(&zram->stats.notify_free);
  3146. index++;
  3147. n -= PAGE_SIZE;
  3148. @@ -973,9 +975,9 @@
  3149. zram = bdev->bd_disk->private_data;
  3150. meta = zram->meta;
  3151. - bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
  3152. + zram_lock_table(&meta->table[index]);
  3153. zram_free_page(zram, index);
  3154. - bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
  3155. + zram_unlock_table(&meta->table[index]);
  3156. atomic64_inc(&zram->stats.notify_free);
  3157. }
  3158. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/block/zram/zram_drv.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/block/zram/zram_drv.h
  3159. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/block/zram/zram_drv.h 2017-04-16 10:37:39.000000000 +0200
  3160. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/block/zram/zram_drv.h 2017-04-18 17:54:22.000000000 +0200
  3161. @@ -73,6 +73,9 @@
  3162. struct zram_table_entry {
  3163. unsigned long handle;
  3164. unsigned long value;
  3165. +#ifdef CONFIG_PREEMPT_RT_BASE
  3166. + spinlock_t lock;
  3167. +#endif
  3168. };
  3169. struct zram_stats {
  3170. @@ -120,4 +123,42 @@
  3171. */
  3172. bool claim; /* Protected by bdev->bd_mutex */
  3173. };
  3174. +
  3175. +#ifndef CONFIG_PREEMPT_RT_BASE
  3176. +static inline void zram_lock_table(struct zram_table_entry *table)
  3177. +{
  3178. + bit_spin_lock(ZRAM_ACCESS, &table->value);
  3179. +}
  3180. +
  3181. +static inline void zram_unlock_table(struct zram_table_entry *table)
  3182. +{
  3183. + bit_spin_unlock(ZRAM_ACCESS, &table->value);
  3184. +}
  3185. +
  3186. +static inline void zram_meta_init_table_locks(struct zram_meta *meta, u64 disksize) { }
  3187. +#else /* CONFIG_PREEMPT_RT_BASE */
  3188. +static inline void zram_lock_table(struct zram_table_entry *table)
  3189. +{
  3190. + spin_lock(&table->lock);
  3191. + __set_bit(ZRAM_ACCESS, &table->value);
  3192. +}
  3193. +
  3194. +static inline void zram_unlock_table(struct zram_table_entry *table)
  3195. +{
  3196. + __clear_bit(ZRAM_ACCESS, &table->value);
  3197. + spin_unlock(&table->lock);
  3198. +}
  3199. +
  3200. +static inline void zram_meta_init_table_locks(struct zram_meta *meta, u64 disksize)
  3201. +{
  3202. + size_t num_pages = disksize >> PAGE_SHIFT;
  3203. + size_t index;
  3204. +
  3205. + for (index = 0; index < num_pages; index++) {
  3206. + spinlock_t *lock = &meta->table[index].lock;
  3207. + spin_lock_init(lock);
  3208. + }
  3209. +}
  3210. +#endif /* CONFIG_PREEMPT_RT_BASE */
  3211. +
  3212. #endif
  3213. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/char/random.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/char/random.c
  3214. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/char/random.c 2017-04-16 10:37:39.000000000 +0200
  3215. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/char/random.c 2017-04-18 17:54:22.000000000 +0200
  3216. @@ -1028,8 +1028,6 @@
  3217. } sample;
  3218. long delta, delta2, delta3;
  3219. - preempt_disable();
  3220. -
  3221. sample.jiffies = jiffies;
  3222. sample.cycles = random_get_entropy();
  3223. sample.num = num;
  3224. @@ -1070,7 +1068,6 @@
  3225. */
  3226. credit_entropy_bits(r, min_t(int, fls(delta>>1), 11));
  3227. }
  3228. - preempt_enable();
  3229. }
  3230. void add_input_randomness(unsigned int type, unsigned int code,
  3231. @@ -1123,28 +1120,27 @@
  3232. return *(ptr + f->reg_idx++);
  3233. }
  3234. -void add_interrupt_randomness(int irq, int irq_flags)
  3235. +void add_interrupt_randomness(int irq, int irq_flags, __u64 ip)
  3236. {
  3237. struct entropy_store *r;
  3238. struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness);
  3239. - struct pt_regs *regs = get_irq_regs();
  3240. unsigned long now = jiffies;
  3241. cycles_t cycles = random_get_entropy();
  3242. __u32 c_high, j_high;
  3243. - __u64 ip;
  3244. unsigned long seed;
  3245. int credit = 0;
  3246. if (cycles == 0)
  3247. - cycles = get_reg(fast_pool, regs);
  3248. + cycles = get_reg(fast_pool, NULL);
  3249. c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0;
  3250. j_high = (sizeof(now) > 4) ? now >> 32 : 0;
  3251. fast_pool->pool[0] ^= cycles ^ j_high ^ irq;
  3252. fast_pool->pool[1] ^= now ^ c_high;
  3253. - ip = regs ? instruction_pointer(regs) : _RET_IP_;
  3254. + if (!ip)
  3255. + ip = _RET_IP_;
  3256. fast_pool->pool[2] ^= ip;
  3257. fast_pool->pool[3] ^= (sizeof(ip) > 4) ? ip >> 32 :
  3258. - get_reg(fast_pool, regs);
  3259. + get_reg(fast_pool, NULL);
  3260. fast_mix(fast_pool);
  3261. add_interrupt_bench(cycles);
  3262. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/clocksource/tcb_clksrc.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/clocksource/tcb_clksrc.c
  3263. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/clocksource/tcb_clksrc.c 2017-04-16 10:37:40.000000000 +0200
  3264. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/clocksource/tcb_clksrc.c 2017-04-18 17:54:22.000000000 +0200
  3265. @@ -23,8 +23,7 @@
  3266. * this 32 bit free-running counter. the second channel is not used.
  3267. *
  3268. * - The third channel may be used to provide a 16-bit clockevent
  3269. - * source, used in either periodic or oneshot mode. This runs
  3270. - * at 32 KiHZ, and can handle delays of up to two seconds.
  3271. + * source, used in either periodic or oneshot mode.
  3272. *
  3273. * A boot clocksource and clockevent source are also currently needed,
  3274. * unless the relevant platforms (ARM/AT91, AVR32/AT32) are changed so
  3275. @@ -74,6 +73,8 @@
  3276. struct tc_clkevt_device {
  3277. struct clock_event_device clkevt;
  3278. struct clk *clk;
  3279. + bool clk_enabled;
  3280. + u32 freq;
  3281. void __iomem *regs;
  3282. };
  3283. @@ -82,15 +83,26 @@
  3284. return container_of(clkevt, struct tc_clkevt_device, clkevt);
  3285. }
  3286. -/* For now, we always use the 32K clock ... this optimizes for NO_HZ,
  3287. - * because using one of the divided clocks would usually mean the
  3288. - * tick rate can never be less than several dozen Hz (vs 0.5 Hz).
  3289. - *
  3290. - * A divided clock could be good for high resolution timers, since
  3291. - * 30.5 usec resolution can seem "low".
  3292. - */
  3293. static u32 timer_clock;
  3294. +static void tc_clk_disable(struct clock_event_device *d)
  3295. +{
  3296. + struct tc_clkevt_device *tcd = to_tc_clkevt(d);
  3297. +
  3298. + clk_disable(tcd->clk);
  3299. + tcd->clk_enabled = false;
  3300. +}
  3301. +
  3302. +static void tc_clk_enable(struct clock_event_device *d)
  3303. +{
  3304. + struct tc_clkevt_device *tcd = to_tc_clkevt(d);
  3305. +
  3306. + if (tcd->clk_enabled)
  3307. + return;
  3308. + clk_enable(tcd->clk);
  3309. + tcd->clk_enabled = true;
  3310. +}
  3311. +
  3312. static int tc_shutdown(struct clock_event_device *d)
  3313. {
  3314. struct tc_clkevt_device *tcd = to_tc_clkevt(d);
  3315. @@ -98,8 +110,14 @@
  3316. __raw_writel(0xff, regs + ATMEL_TC_REG(2, IDR));
  3317. __raw_writel(ATMEL_TC_CLKDIS, regs + ATMEL_TC_REG(2, CCR));
  3318. + return 0;
  3319. +}
  3320. +
  3321. +static int tc_shutdown_clk_off(struct clock_event_device *d)
  3322. +{
  3323. + tc_shutdown(d);
  3324. if (!clockevent_state_detached(d))
  3325. - clk_disable(tcd->clk);
  3326. + tc_clk_disable(d);
  3327. return 0;
  3328. }
  3329. @@ -112,9 +130,9 @@
  3330. if (clockevent_state_oneshot(d) || clockevent_state_periodic(d))
  3331. tc_shutdown(d);
  3332. - clk_enable(tcd->clk);
  3333. + tc_clk_enable(d);
  3334. - /* slow clock, count up to RC, then irq and stop */
  3335. + /* count up to RC, then irq and stop */
  3336. __raw_writel(timer_clock | ATMEL_TC_CPCSTOP | ATMEL_TC_WAVE |
  3337. ATMEL_TC_WAVESEL_UP_AUTO, regs + ATMEL_TC_REG(2, CMR));
  3338. __raw_writel(ATMEL_TC_CPCS, regs + ATMEL_TC_REG(2, IER));
  3339. @@ -134,12 +152,12 @@
  3340. /* By not making the gentime core emulate periodic mode on top
  3341. * of oneshot, we get lower overhead and improved accuracy.
  3342. */
  3343. - clk_enable(tcd->clk);
  3344. + tc_clk_enable(d);
  3345. - /* slow clock, count up to RC, then irq and restart */
  3346. + /* count up to RC, then irq and restart */
  3347. __raw_writel(timer_clock | ATMEL_TC_WAVE | ATMEL_TC_WAVESEL_UP_AUTO,
  3348. regs + ATMEL_TC_REG(2, CMR));
  3349. - __raw_writel((32768 + HZ / 2) / HZ, tcaddr + ATMEL_TC_REG(2, RC));
  3350. + __raw_writel((tcd->freq + HZ / 2) / HZ, tcaddr + ATMEL_TC_REG(2, RC));
  3351. /* Enable clock and interrupts on RC compare */
  3352. __raw_writel(ATMEL_TC_CPCS, regs + ATMEL_TC_REG(2, IER));
  3353. @@ -166,9 +184,13 @@
  3354. .features = CLOCK_EVT_FEAT_PERIODIC |
  3355. CLOCK_EVT_FEAT_ONESHOT,
  3356. /* Should be lower than at91rm9200's system timer */
  3357. +#ifdef CONFIG_ATMEL_TCB_CLKSRC_USE_SLOW_CLOCK
  3358. .rating = 125,
  3359. +#else
  3360. + .rating = 200,
  3361. +#endif
  3362. .set_next_event = tc_next_event,
  3363. - .set_state_shutdown = tc_shutdown,
  3364. + .set_state_shutdown = tc_shutdown_clk_off,
  3365. .set_state_periodic = tc_set_periodic,
  3366. .set_state_oneshot = tc_set_oneshot,
  3367. },
  3368. @@ -188,8 +210,9 @@
  3369. return IRQ_NONE;
  3370. }
  3371. -static int __init setup_clkevents(struct atmel_tc *tc, int clk32k_divisor_idx)
  3372. +static int __init setup_clkevents(struct atmel_tc *tc, int divisor_idx)
  3373. {
  3374. + unsigned divisor = atmel_tc_divisors[divisor_idx];
  3375. int ret;
  3376. struct clk *t2_clk = tc->clk[2];
  3377. int irq = tc->irq[2];
  3378. @@ -210,7 +233,11 @@
  3379. clkevt.regs = tc->regs;
  3380. clkevt.clk = t2_clk;
  3381. - timer_clock = clk32k_divisor_idx;
  3382. + timer_clock = divisor_idx;
  3383. + if (!divisor)
  3384. + clkevt.freq = 32768;
  3385. + else
  3386. + clkevt.freq = clk_get_rate(t2_clk) / divisor;
  3387. clkevt.clkevt.cpumask = cpumask_of(0);
  3388. @@ -221,7 +248,7 @@
  3389. return ret;
  3390. }
  3391. - clockevents_config_and_register(&clkevt.clkevt, 32768, 1, 0xffff);
  3392. + clockevents_config_and_register(&clkevt.clkevt, clkevt.freq, 1, 0xffff);
  3393. return ret;
  3394. }
  3395. @@ -358,7 +385,11 @@
  3396. goto err_disable_t1;
  3397. /* channel 2: periodic and oneshot timer support */
  3398. +#ifdef CONFIG_ATMEL_TCB_CLKSRC_USE_SLOW_CLOCK
  3399. ret = setup_clkevents(tc, clk32k_divisor_idx);
  3400. +#else
  3401. + ret = setup_clkevents(tc, best_divisor_idx);
  3402. +#endif
  3403. if (ret)
  3404. goto err_unregister_clksrc;
  3405. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/clocksource/timer-atmel-pit.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/clocksource/timer-atmel-pit.c
  3406. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/clocksource/timer-atmel-pit.c 2017-04-16 10:37:40.000000000 +0200
  3407. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/clocksource/timer-atmel-pit.c 2017-04-18 17:54:22.000000000 +0200
  3408. @@ -46,6 +46,7 @@
  3409. u32 cycle;
  3410. u32 cnt;
  3411. unsigned int irq;
  3412. + bool irq_requested;
  3413. struct clk *mck;
  3414. };
  3415. @@ -96,15 +97,29 @@
  3416. /* disable irq, leaving the clocksource active */
  3417. pit_write(data->base, AT91_PIT_MR, (data->cycle - 1) | AT91_PIT_PITEN);
  3418. + if (data->irq_requested) {
  3419. + free_irq(data->irq, data);
  3420. + data->irq_requested = false;
  3421. + }
  3422. return 0;
  3423. }
  3424. +static irqreturn_t at91sam926x_pit_interrupt(int irq, void *dev_id);
  3425. /*
  3426. * Clockevent device: interrupts every 1/HZ (== pit_cycles * MCK/16)
  3427. */
  3428. static int pit_clkevt_set_periodic(struct clock_event_device *dev)
  3429. {
  3430. struct pit_data *data = clkevt_to_pit_data(dev);
  3431. + int ret;
  3432. +
  3433. + ret = request_irq(data->irq, at91sam926x_pit_interrupt,
  3434. + IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL,
  3435. + "at91_tick", data);
  3436. + if (ret)
  3437. + panic(pr_fmt("Unable to setup IRQ\n"));
  3438. +
  3439. + data->irq_requested = true;
  3440. /* update clocksource counter */
  3441. data->cnt += data->cycle * PIT_PICNT(pit_read(data->base, AT91_PIT_PIVR));
  3442. @@ -230,15 +245,6 @@
  3443. return ret;
  3444. }
  3445. - /* Set up irq handler */
  3446. - ret = request_irq(data->irq, at91sam926x_pit_interrupt,
  3447. - IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL,
  3448. - "at91_tick", data);
  3449. - if (ret) {
  3450. - pr_err("Unable to setup IRQ\n");
  3451. - return ret;
  3452. - }
  3453. -
  3454. /* Set up and register clockevents */
  3455. data->clkevt.name = "pit";
  3456. data->clkevt.features = CLOCK_EVT_FEAT_PERIODIC;
  3457. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/clocksource/timer-atmel-st.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/clocksource/timer-atmel-st.c
  3458. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/clocksource/timer-atmel-st.c 2017-04-16 10:37:40.000000000 +0200
  3459. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/clocksource/timer-atmel-st.c 2017-04-18 17:54:22.000000000 +0200
  3460. @@ -115,18 +115,29 @@
  3461. last_crtr = read_CRTR();
  3462. }
  3463. +static int atmel_st_irq;
  3464. +
  3465. static int clkevt32k_shutdown(struct clock_event_device *evt)
  3466. {
  3467. clkdev32k_disable_and_flush_irq();
  3468. irqmask = 0;
  3469. regmap_write(regmap_st, AT91_ST_IER, irqmask);
  3470. + free_irq(atmel_st_irq, regmap_st);
  3471. return 0;
  3472. }
  3473. static int clkevt32k_set_oneshot(struct clock_event_device *dev)
  3474. {
  3475. + int ret;
  3476. +
  3477. clkdev32k_disable_and_flush_irq();
  3478. + ret = request_irq(atmel_st_irq, at91rm9200_timer_interrupt,
  3479. + IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL,
  3480. + "at91_tick", regmap_st);
  3481. + if (ret)
  3482. + panic(pr_fmt("Unable to setup IRQ\n"));
  3483. +
  3484. /*
  3485. * ALM for oneshot irqs, set by next_event()
  3486. * before 32 seconds have passed.
  3487. @@ -139,8 +150,16 @@
  3488. static int clkevt32k_set_periodic(struct clock_event_device *dev)
  3489. {
  3490. + int ret;
  3491. +
  3492. clkdev32k_disable_and_flush_irq();
  3493. + ret = request_irq(atmel_st_irq, at91rm9200_timer_interrupt,
  3494. + IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL,
  3495. + "at91_tick", regmap_st);
  3496. + if (ret)
  3497. + panic(pr_fmt("Unable to setup IRQ\n"));
  3498. +
  3499. /* PIT for periodic irqs; fixed rate of 1/HZ */
  3500. irqmask = AT91_ST_PITS;
  3501. regmap_write(regmap_st, AT91_ST_PIMR, timer_latch);
  3502. @@ -198,7 +217,7 @@
  3503. {
  3504. struct clk *sclk;
  3505. unsigned int sclk_rate, val;
  3506. - int irq, ret;
  3507. + int ret;
  3508. regmap_st = syscon_node_to_regmap(node);
  3509. if (IS_ERR(regmap_st)) {
  3510. @@ -212,21 +231,12 @@
  3511. regmap_read(regmap_st, AT91_ST_SR, &val);
  3512. /* Get the interrupts property */
  3513. - irq = irq_of_parse_and_map(node, 0);
  3514. - if (!irq) {
  3515. + atmel_st_irq = irq_of_parse_and_map(node, 0);
  3516. + if (!atmel_st_irq) {
  3517. pr_err("Unable to get IRQ from DT\n");
  3518. return -EINVAL;
  3519. }
  3520. - /* Make IRQs happen for the system timer */
  3521. - ret = request_irq(irq, at91rm9200_timer_interrupt,
  3522. - IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL,
  3523. - "at91_tick", regmap_st);
  3524. - if (ret) {
  3525. - pr_err("Unable to setup IRQ\n");
  3526. - return ret;
  3527. - }
  3528. -
  3529. sclk = of_clk_get(node, 0);
  3530. if (IS_ERR(sclk)) {
  3531. pr_err("Unable to get slow clock\n");
  3532. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/connector/cn_proc.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/connector/cn_proc.c
  3533. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/connector/cn_proc.c 2017-04-16 10:37:40.000000000 +0200
  3534. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/connector/cn_proc.c 2017-04-18 17:54:22.000000000 +0200
  3535. @@ -32,6 +32,7 @@
  3536. #include <linux/pid_namespace.h>
  3537. #include <linux/cn_proc.h>
  3538. +#include <linux/locallock.h>
  3539. /*
  3540. * Size of a cn_msg followed by a proc_event structure. Since the
  3541. @@ -54,10 +55,11 @@
  3542. /* proc_event_counts is used as the sequence number of the netlink message */
  3543. static DEFINE_PER_CPU(__u32, proc_event_counts) = { 0 };
  3544. +static DEFINE_LOCAL_IRQ_LOCK(send_msg_lock);
  3545. static inline void send_msg(struct cn_msg *msg)
  3546. {
  3547. - preempt_disable();
  3548. + local_lock(send_msg_lock);
  3549. msg->seq = __this_cpu_inc_return(proc_event_counts) - 1;
  3550. ((struct proc_event *)msg->data)->cpu = smp_processor_id();
  3551. @@ -70,7 +72,7 @@
  3552. */
  3553. cn_netlink_send(msg, 0, CN_IDX_PROC, GFP_NOWAIT);
  3554. - preempt_enable();
  3555. + local_unlock(send_msg_lock);
  3556. }
  3557. void proc_fork_connector(struct task_struct *task)
  3558. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/cpufreq/Kconfig.x86 linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/cpufreq/Kconfig.x86
  3559. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/cpufreq/Kconfig.x86 2017-04-16 10:37:40.000000000 +0200
  3560. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/cpufreq/Kconfig.x86 2017-04-18 17:54:22.000000000 +0200
  3561. @@ -124,7 +124,7 @@
  3562. config X86_POWERNOW_K8
  3563. tristate "AMD Opteron/Athlon64 PowerNow!"
  3564. - depends on ACPI && ACPI_PROCESSOR && X86_ACPI_CPUFREQ
  3565. + depends on ACPI && ACPI_PROCESSOR && X86_ACPI_CPUFREQ && !PREEMPT_RT_BASE
  3566. help
  3567. This adds the CPUFreq driver for K8/early Opteron/Athlon64 processors.
  3568. Support for K10 and newer processors is now in acpi-cpufreq.
  3569. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
  3570. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2017-04-16 10:37:42.000000000 +0200
  3571. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2017-04-18 17:54:22.000000000 +0200
  3572. @@ -1537,7 +1537,9 @@
  3573. if (ret)
  3574. return ret;
  3575. +#ifndef CONFIG_PREEMPT_RT_BASE
  3576. trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags);
  3577. +#endif
  3578. i915_gem_execbuffer_move_to_active(vmas, params->request);
  3579. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/i915/i915_gem_shrinker.c
  3580. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c 2017-04-16 10:37:42.000000000 +0200
  3581. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/i915/i915_gem_shrinker.c 2017-04-18 17:54:22.000000000 +0200
  3582. @@ -40,7 +40,7 @@
  3583. if (!mutex_is_locked(mutex))
  3584. return false;
  3585. -#if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_MUTEX_SPIN_ON_OWNER)
  3586. +#if (defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_MUTEX_SPIN_ON_OWNER)) && !defined(CONFIG_PREEMPT_RT_BASE)
  3587. return mutex->owner == task;
  3588. #else
  3589. /* Since UP may be pre-empted, we cannot assume that we own the lock */
  3590. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/i915/i915_irq.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/i915/i915_irq.c
  3591. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/i915/i915_irq.c 2017-04-16 10:37:42.000000000 +0200
  3592. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/i915/i915_irq.c 2017-04-18 17:54:22.000000000 +0200
  3593. @@ -812,6 +812,7 @@
  3594. spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
  3595. /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
  3596. + preempt_disable_rt();
  3597. /* Get optional system timestamp before query. */
  3598. if (stime)
  3599. @@ -863,6 +864,7 @@
  3600. *etime = ktime_get();
  3601. /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
  3602. + preempt_enable_rt();
  3603. spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
  3604. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/i915/intel_display.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/i915/intel_display.c
  3605. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/i915/intel_display.c 2017-04-16 10:37:42.000000000 +0200
  3606. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/i915/intel_display.c 2017-04-18 17:54:22.000000000 +0200
  3607. @@ -12141,7 +12141,7 @@
  3608. struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
  3609. struct intel_flip_work *work;
  3610. - WARN_ON(!in_interrupt());
  3611. + WARN_ON_NONRT(!in_interrupt());
  3612. if (crtc == NULL)
  3613. return;
  3614. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/i915/intel_sprite.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/i915/intel_sprite.c
  3615. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/i915/intel_sprite.c 2017-04-16 10:37:42.000000000 +0200
  3616. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/i915/intel_sprite.c 2017-04-18 17:54:22.000000000 +0200
  3617. @@ -35,6 +35,7 @@
  3618. #include <drm/drm_rect.h>
  3619. #include <drm/drm_atomic.h>
  3620. #include <drm/drm_plane_helper.h>
  3621. +#include <linux/locallock.h>
  3622. #include "intel_drv.h"
  3623. #include "intel_frontbuffer.h"
  3624. #include <drm/i915_drm.h>
  3625. @@ -65,6 +66,8 @@
  3626. 1000 * adjusted_mode->crtc_htotal);
  3627. }
  3628. +static DEFINE_LOCAL_IRQ_LOCK(pipe_update_lock);
  3629. +
  3630. /**
  3631. * intel_pipe_update_start() - start update of a set of display registers
  3632. * @crtc: the crtc of which the registers are going to be updated
  3633. @@ -95,7 +98,7 @@
  3634. min = vblank_start - intel_usecs_to_scanlines(adjusted_mode, 100);
  3635. max = vblank_start - 1;
  3636. - local_irq_disable();
  3637. + local_lock_irq(pipe_update_lock);
  3638. if (min <= 0 || max <= 0)
  3639. return;
  3640. @@ -125,11 +128,11 @@
  3641. break;
  3642. }
  3643. - local_irq_enable();
  3644. + local_unlock_irq(pipe_update_lock);
  3645. timeout = schedule_timeout(timeout);
  3646. - local_irq_disable();
  3647. + local_lock_irq(pipe_update_lock);
  3648. }
  3649. finish_wait(wq, &wait);
  3650. @@ -181,7 +184,7 @@
  3651. crtc->base.state->event = NULL;
  3652. }
  3653. - local_irq_enable();
  3654. + local_unlock_irq(pipe_update_lock);
  3655. if (crtc->debug.start_vbl_count &&
  3656. crtc->debug.start_vbl_count != end_vbl_count) {
  3657. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/msm/msm_gem_shrinker.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/msm/msm_gem_shrinker.c
  3658. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/msm/msm_gem_shrinker.c 2017-04-16 10:37:42.000000000 +0200
  3659. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/msm/msm_gem_shrinker.c 2017-04-18 17:54:22.000000000 +0200
  3660. @@ -23,7 +23,7 @@
  3661. if (!mutex_is_locked(mutex))
  3662. return false;
  3663. -#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
  3664. +#if (defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)) && !defined(CONFIG_PREEMPT_RT_BASE)
  3665. return mutex->owner == task;
  3666. #else
  3667. /* Since UP may be pre-empted, we cannot assume that we own the lock */
  3668. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/radeon/radeon_display.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/radeon/radeon_display.c
  3669. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/gpu/drm/radeon/radeon_display.c 2017-04-16 10:37:43.000000000 +0200
  3670. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/gpu/drm/radeon/radeon_display.c 2017-04-18 17:54:23.000000000 +0200
  3671. @@ -1845,6 +1845,7 @@
  3672. struct radeon_device *rdev = dev->dev_private;
  3673. /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
  3674. + preempt_disable_rt();
  3675. /* Get optional system timestamp before query. */
  3676. if (stime)
  3677. @@ -1937,6 +1938,7 @@
  3678. *etime = ktime_get();
  3679. /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
  3680. + preempt_enable_rt();
  3681. /* Decode into vertical and horizontal scanout position. */
  3682. *vpos = position & 0x1fff;
  3683. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/hv/vmbus_drv.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/hv/vmbus_drv.c
  3684. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/hv/vmbus_drv.c 2017-04-16 10:37:44.000000000 +0200
  3685. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/hv/vmbus_drv.c 2017-04-18 17:54:23.000000000 +0200
  3686. @@ -761,6 +761,8 @@
  3687. void *page_addr;
  3688. struct hv_message *msg;
  3689. union hv_synic_event_flags *event;
  3690. + struct pt_regs *regs = get_irq_regs();
  3691. + u64 ip = regs ? instruction_pointer(regs) : 0;
  3692. bool handled = false;
  3693. page_addr = hv_context.synic_event_page[cpu];
  3694. @@ -808,7 +810,7 @@
  3695. tasklet_schedule(hv_context.msg_dpc[cpu]);
  3696. }
  3697. - add_interrupt_randomness(HYPERVISOR_CALLBACK_VECTOR, 0);
  3698. + add_interrupt_randomness(HYPERVISOR_CALLBACK_VECTOR, 0, ip);
  3699. }
  3700. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/alim15x3.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/alim15x3.c
  3701. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/alim15x3.c 2017-04-16 10:37:44.000000000 +0200
  3702. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/alim15x3.c 2017-04-18 17:54:23.000000000 +0200
  3703. @@ -234,7 +234,7 @@
  3704. isa_dev = pci_get_device(PCI_VENDOR_ID_AL, PCI_DEVICE_ID_AL_M1533, NULL);
  3705. - local_irq_save(flags);
  3706. + local_irq_save_nort(flags);
  3707. if (m5229_revision < 0xC2) {
  3708. /*
  3709. @@ -325,7 +325,7 @@
  3710. }
  3711. pci_dev_put(north);
  3712. pci_dev_put(isa_dev);
  3713. - local_irq_restore(flags);
  3714. + local_irq_restore_nort(flags);
  3715. return 0;
  3716. }
  3717. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/hpt366.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/hpt366.c
  3718. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/hpt366.c 2017-04-16 10:37:44.000000000 +0200
  3719. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/hpt366.c 2017-04-18 17:54:23.000000000 +0200
  3720. @@ -1236,7 +1236,7 @@
  3721. dma_old = inb(base + 2);
  3722. - local_irq_save(flags);
  3723. + local_irq_save_nort(flags);
  3724. dma_new = dma_old;
  3725. pci_read_config_byte(dev, hwif->channel ? 0x4b : 0x43, &masterdma);
  3726. @@ -1247,7 +1247,7 @@
  3727. if (dma_new != dma_old)
  3728. outb(dma_new, base + 2);
  3729. - local_irq_restore(flags);
  3730. + local_irq_restore_nort(flags);
  3731. printk(KERN_INFO " %s: BM-DMA at 0x%04lx-0x%04lx\n",
  3732. hwif->name, base, base + 7);
  3733. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/ide-io-std.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/ide-io-std.c
  3734. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/ide-io-std.c 2017-04-16 10:37:44.000000000 +0200
  3735. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/ide-io-std.c 2017-04-18 17:54:23.000000000 +0200
  3736. @@ -175,7 +175,7 @@
  3737. unsigned long uninitialized_var(flags);
  3738. if ((io_32bit & 2) && !mmio) {
  3739. - local_irq_save(flags);
  3740. + local_irq_save_nort(flags);
  3741. ata_vlb_sync(io_ports->nsect_addr);
  3742. }
  3743. @@ -186,7 +186,7 @@
  3744. insl(data_addr, buf, words);
  3745. if ((io_32bit & 2) && !mmio)
  3746. - local_irq_restore(flags);
  3747. + local_irq_restore_nort(flags);
  3748. if (((len + 1) & 3) < 2)
  3749. return;
  3750. @@ -219,7 +219,7 @@
  3751. unsigned long uninitialized_var(flags);
  3752. if ((io_32bit & 2) && !mmio) {
  3753. - local_irq_save(flags);
  3754. + local_irq_save_nort(flags);
  3755. ata_vlb_sync(io_ports->nsect_addr);
  3756. }
  3757. @@ -230,7 +230,7 @@
  3758. outsl(data_addr, buf, words);
  3759. if ((io_32bit & 2) && !mmio)
  3760. - local_irq_restore(flags);
  3761. + local_irq_restore_nort(flags);
  3762. if (((len + 1) & 3) < 2)
  3763. return;
  3764. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/ide-io.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/ide-io.c
  3765. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/ide-io.c 2017-04-16 10:37:44.000000000 +0200
  3766. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/ide-io.c 2017-04-18 17:54:23.000000000 +0200
  3767. @@ -659,7 +659,7 @@
  3768. /* disable_irq_nosync ?? */
  3769. disable_irq(hwif->irq);
  3770. /* local CPU only, as if we were handling an interrupt */
  3771. - local_irq_disable();
  3772. + local_irq_disable_nort();
  3773. if (hwif->polling) {
  3774. startstop = handler(drive);
  3775. } else if (drive_is_ready(drive)) {
  3776. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/ide-iops.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/ide-iops.c
  3777. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/ide-iops.c 2017-04-16 10:37:44.000000000 +0200
  3778. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/ide-iops.c 2017-04-18 17:54:23.000000000 +0200
  3779. @@ -129,12 +129,12 @@
  3780. if ((stat & ATA_BUSY) == 0)
  3781. break;
  3782. - local_irq_restore(flags);
  3783. + local_irq_restore_nort(flags);
  3784. *rstat = stat;
  3785. return -EBUSY;
  3786. }
  3787. }
  3788. - local_irq_restore(flags);
  3789. + local_irq_restore_nort(flags);
  3790. }
  3791. /*
  3792. * Allow status to settle, then read it again.
  3793. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/ide-probe.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/ide-probe.c
  3794. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/ide-probe.c 2017-04-16 10:37:44.000000000 +0200
  3795. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/ide-probe.c 2017-04-18 17:54:23.000000000 +0200
  3796. @@ -196,10 +196,10 @@
  3797. int bswap = 1;
  3798. /* local CPU only; some systems need this */
  3799. - local_irq_save(flags);
  3800. + local_irq_save_nort(flags);
  3801. /* read 512 bytes of id info */
  3802. hwif->tp_ops->input_data(drive, NULL, id, SECTOR_SIZE);
  3803. - local_irq_restore(flags);
  3804. + local_irq_restore_nort(flags);
  3805. drive->dev_flags |= IDE_DFLAG_ID_READ;
  3806. #ifdef DEBUG
  3807. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/ide-taskfile.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/ide-taskfile.c
  3808. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/ide/ide-taskfile.c 2017-04-16 10:37:44.000000000 +0200
  3809. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/ide/ide-taskfile.c 2017-04-18 17:54:23.000000000 +0200
  3810. @@ -250,7 +250,7 @@
  3811. page_is_high = PageHighMem(page);
  3812. if (page_is_high)
  3813. - local_irq_save(flags);
  3814. + local_irq_save_nort(flags);
  3815. buf = kmap_atomic(page) + offset;
  3816. @@ -271,7 +271,7 @@
  3817. kunmap_atomic(buf);
  3818. if (page_is_high)
  3819. - local_irq_restore(flags);
  3820. + local_irq_restore_nort(flags);
  3821. len -= nr_bytes;
  3822. }
  3823. @@ -414,7 +414,7 @@
  3824. }
  3825. if ((drive->dev_flags & IDE_DFLAG_UNMASK) == 0)
  3826. - local_irq_disable();
  3827. + local_irq_disable_nort();
  3828. ide_set_handler(drive, &task_pio_intr, WAIT_WORSTCASE);
  3829. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
  3830. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2017-04-16 10:37:45.000000000 +0200
  3831. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2017-04-18 17:54:23.000000000 +0200
  3832. @@ -902,7 +902,7 @@
  3833. ipoib_dbg_mcast(priv, "restarting multicast task\n");
  3834. - local_irq_save(flags);
  3835. + local_irq_save_nort(flags);
  3836. netif_addr_lock(dev);
  3837. spin_lock(&priv->lock);
  3838. @@ -984,7 +984,7 @@
  3839. spin_unlock(&priv->lock);
  3840. netif_addr_unlock(dev);
  3841. - local_irq_restore(flags);
  3842. + local_irq_restore_nort(flags);
  3843. /*
  3844. * make sure the in-flight joins have finished before we attempt
  3845. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/input/gameport/gameport.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/input/gameport/gameport.c
  3846. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/input/gameport/gameport.c 2017-04-16 10:37:45.000000000 +0200
  3847. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/input/gameport/gameport.c 2017-04-18 17:54:23.000000000 +0200
  3848. @@ -91,13 +91,13 @@
  3849. tx = ~0;
  3850. for (i = 0; i < 50; i++) {
  3851. - local_irq_save(flags);
  3852. + local_irq_save_nort(flags);
  3853. t1 = ktime_get_ns();
  3854. for (t = 0; t < 50; t++)
  3855. gameport_read(gameport);
  3856. t2 = ktime_get_ns();
  3857. t3 = ktime_get_ns();
  3858. - local_irq_restore(flags);
  3859. + local_irq_restore_nort(flags);
  3860. udelay(i * 10);
  3861. t = (t2 - t1) - (t3 - t2);
  3862. if (t < tx)
  3863. @@ -124,12 +124,12 @@
  3864. tx = 1 << 30;
  3865. for(i = 0; i < 50; i++) {
  3866. - local_irq_save(flags);
  3867. + local_irq_save_nort(flags);
  3868. GET_TIME(t1);
  3869. for (t = 0; t < 50; t++) gameport_read(gameport);
  3870. GET_TIME(t2);
  3871. GET_TIME(t3);
  3872. - local_irq_restore(flags);
  3873. + local_irq_restore_nort(flags);
  3874. udelay(i * 10);
  3875. if ((t = DELTA(t2,t1) - DELTA(t3,t2)) < tx) tx = t;
  3876. }
  3877. @@ -148,11 +148,11 @@
  3878. tx = 1 << 30;
  3879. for(i = 0; i < 50; i++) {
  3880. - local_irq_save(flags);
  3881. + local_irq_save_nort(flags);
  3882. t1 = rdtsc();
  3883. for (t = 0; t < 50; t++) gameport_read(gameport);
  3884. t2 = rdtsc();
  3885. - local_irq_restore(flags);
  3886. + local_irq_restore_nort(flags);
  3887. udelay(i * 10);
  3888. if (t2 - t1 < tx) tx = t2 - t1;
  3889. }
  3890. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/iommu/amd_iommu.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/iommu/amd_iommu.c
  3891. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/iommu/amd_iommu.c 2017-04-16 10:37:46.000000000 +0200
  3892. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/iommu/amd_iommu.c 2017-04-18 17:54:23.000000000 +0200
  3893. @@ -1923,10 +1923,10 @@
  3894. int ret;
  3895. /*
  3896. - * Must be called with IRQs disabled. Warn here to detect early
  3897. - * when its not.
  3898. + * Must be called with IRQs disabled on a non RT kernel. Warn here to
  3899. + * detect early when its not.
  3900. */
  3901. - WARN_ON(!irqs_disabled());
  3902. + WARN_ON_NONRT(!irqs_disabled());
  3903. /* lock domain */
  3904. spin_lock(&domain->lock);
  3905. @@ -2094,10 +2094,10 @@
  3906. struct protection_domain *domain;
  3907. /*
  3908. - * Must be called with IRQs disabled. Warn here to detect early
  3909. - * when its not.
  3910. + * Must be called with IRQs disabled on a non RT kernel. Warn here to
  3911. + * detect early when its not.
  3912. */
  3913. - WARN_ON(!irqs_disabled());
  3914. + WARN_ON_NONRT(!irqs_disabled());
  3915. if (WARN_ON(!dev_data->domain))
  3916. return;
  3917. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/iommu/intel-iommu.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/iommu/intel-iommu.c
  3918. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/iommu/intel-iommu.c 2017-04-16 10:37:46.000000000 +0200
  3919. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/iommu/intel-iommu.c 2017-04-18 17:54:23.000000000 +0200
  3920. @@ -479,7 +479,7 @@
  3921. struct deferred_flush_table *tables;
  3922. };
  3923. -DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
  3924. +static DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
  3925. /* bitmap for indexing intel_iommus */
  3926. static int g_num_of_iommus;
  3927. @@ -3716,10 +3716,8 @@
  3928. struct intel_iommu *iommu;
  3929. struct deferred_flush_entry *entry;
  3930. struct deferred_flush_data *flush_data;
  3931. - unsigned int cpuid;
  3932. - cpuid = get_cpu();
  3933. - flush_data = per_cpu_ptr(&deferred_flush, cpuid);
  3934. + flush_data = raw_cpu_ptr(&deferred_flush);
  3935. /* Flush all CPUs' entries to avoid deferring too much. If
  3936. * this becomes a bottleneck, can just flush us, and rely on
  3937. @@ -3752,8 +3750,6 @@
  3938. }
  3939. flush_data->size++;
  3940. spin_unlock_irqrestore(&flush_data->lock, flags);
  3941. -
  3942. - put_cpu();
  3943. }
  3944. static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
  3945. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/iommu/iova.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/iommu/iova.c
  3946. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/iommu/iova.c 2017-04-16 10:37:46.000000000 +0200
  3947. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/iommu/iova.c 2017-04-18 17:54:23.000000000 +0200
  3948. @@ -22,6 +22,7 @@
  3949. #include <linux/slab.h>
  3950. #include <linux/smp.h>
  3951. #include <linux/bitops.h>
  3952. +#include <linux/cpu.h>
  3953. static bool iova_rcache_insert(struct iova_domain *iovad,
  3954. unsigned long pfn,
  3955. @@ -420,10 +421,8 @@
  3956. /* Try replenishing IOVAs by flushing rcache. */
  3957. flushed_rcache = true;
  3958. - preempt_disable();
  3959. for_each_online_cpu(cpu)
  3960. free_cpu_cached_iovas(cpu, iovad);
  3961. - preempt_enable();
  3962. goto retry;
  3963. }
  3964. @@ -751,7 +750,7 @@
  3965. bool can_insert = false;
  3966. unsigned long flags;
  3967. - cpu_rcache = get_cpu_ptr(rcache->cpu_rcaches);
  3968. + cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
  3969. spin_lock_irqsave(&cpu_rcache->lock, flags);
  3970. if (!iova_magazine_full(cpu_rcache->loaded)) {
  3971. @@ -781,7 +780,6 @@
  3972. iova_magazine_push(cpu_rcache->loaded, iova_pfn);
  3973. spin_unlock_irqrestore(&cpu_rcache->lock, flags);
  3974. - put_cpu_ptr(rcache->cpu_rcaches);
  3975. if (mag_to_free) {
  3976. iova_magazine_free_pfns(mag_to_free, iovad);
  3977. @@ -815,7 +813,7 @@
  3978. bool has_pfn = false;
  3979. unsigned long flags;
  3980. - cpu_rcache = get_cpu_ptr(rcache->cpu_rcaches);
  3981. + cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
  3982. spin_lock_irqsave(&cpu_rcache->lock, flags);
  3983. if (!iova_magazine_empty(cpu_rcache->loaded)) {
  3984. @@ -837,7 +835,6 @@
  3985. iova_pfn = iova_magazine_pop(cpu_rcache->loaded, limit_pfn);
  3986. spin_unlock_irqrestore(&cpu_rcache->lock, flags);
  3987. - put_cpu_ptr(rcache->cpu_rcaches);
  3988. return iova_pfn;
  3989. }
  3990. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/leds/trigger/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/leds/trigger/Kconfig
  3991. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/leds/trigger/Kconfig 2017-04-16 10:37:47.000000000 +0200
  3992. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/leds/trigger/Kconfig 2017-04-18 17:54:23.000000000 +0200
  3993. @@ -69,7 +69,7 @@
  3994. config LEDS_TRIGGER_CPU
  3995. bool "LED CPU Trigger"
  3996. - depends on LEDS_TRIGGERS
  3997. + depends on LEDS_TRIGGERS && !PREEMPT_RT_BASE
  3998. help
  3999. This allows LEDs to be controlled by active CPUs. This shows
  4000. the active CPUs across an array of LEDs so you can see which
  4001. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/md/bcache/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/md/bcache/Kconfig
  4002. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/md/bcache/Kconfig 2017-04-16 10:37:47.000000000 +0200
  4003. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/md/bcache/Kconfig 2017-04-18 17:54:23.000000000 +0200
  4004. @@ -1,6 +1,7 @@
  4005. config BCACHE
  4006. tristate "Block device as cache"
  4007. + depends on !PREEMPT_RT_FULL
  4008. ---help---
  4009. Allows a block device to be used as cache for other devices; uses
  4010. a btree for indexing and the layout is optimized for SSDs.
  4011. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/md/dm-rq.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/md/dm-rq.c
  4012. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/md/dm-rq.c 2017-04-16 10:37:47.000000000 +0200
  4013. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/md/dm-rq.c 2017-04-18 17:54:23.000000000 +0200
  4014. @@ -842,7 +842,7 @@
  4015. /* Establish tio->ti before queuing work (map_tio_request) */
  4016. tio->ti = ti;
  4017. kthread_queue_work(&md->kworker, &tio->work);
  4018. - BUG_ON(!irqs_disabled());
  4019. + BUG_ON_NONRT(!irqs_disabled());
  4020. }
  4021. }
  4022. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/md/raid5.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/md/raid5.c
  4023. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/md/raid5.c 2017-04-16 10:37:47.000000000 +0200
  4024. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/md/raid5.c 2017-04-18 17:54:23.000000000 +0200
  4025. @@ -1928,8 +1928,9 @@
  4026. struct raid5_percpu *percpu;
  4027. unsigned long cpu;
  4028. - cpu = get_cpu();
  4029. + cpu = get_cpu_light();
  4030. percpu = per_cpu_ptr(conf->percpu, cpu);
  4031. + spin_lock(&percpu->lock);
  4032. if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
  4033. ops_run_biofill(sh);
  4034. overlap_clear++;
  4035. @@ -1985,7 +1986,8 @@
  4036. if (test_and_clear_bit(R5_Overlap, &dev->flags))
  4037. wake_up(&sh->raid_conf->wait_for_overlap);
  4038. }
  4039. - put_cpu();
  4040. + spin_unlock(&percpu->lock);
  4041. + put_cpu_light();
  4042. }
  4043. static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp,
  4044. @@ -6391,6 +6393,7 @@
  4045. __func__, cpu);
  4046. return -ENOMEM;
  4047. }
  4048. + spin_lock_init(&per_cpu_ptr(conf->percpu, cpu)->lock);
  4049. return 0;
  4050. }
  4051. @@ -6401,7 +6404,6 @@
  4052. conf->percpu = alloc_percpu(struct raid5_percpu);
  4053. if (!conf->percpu)
  4054. return -ENOMEM;
  4055. -
  4056. err = cpuhp_state_add_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);
  4057. if (!err) {
  4058. conf->scribble_disks = max(conf->raid_disks,
  4059. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/md/raid5.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/md/raid5.h
  4060. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/md/raid5.h 2017-04-16 10:37:47.000000000 +0200
  4061. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/md/raid5.h 2017-04-18 17:54:23.000000000 +0200
  4062. @@ -504,6 +504,7 @@
  4063. int recovery_disabled;
  4064. /* per cpu variables */
  4065. struct raid5_percpu {
  4066. + spinlock_t lock; /* Protection for -RT */
  4067. struct page *spare_page; /* Used when checking P/Q in raid6 */
  4068. struct flex_array *scribble; /* space for constructing buffer
  4069. * lists and performing address
  4070. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/misc/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/misc/Kconfig
  4071. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/misc/Kconfig 2017-04-16 10:37:53.000000000 +0200
  4072. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/misc/Kconfig 2017-04-18 17:54:23.000000000 +0200
  4073. @@ -62,6 +62,7 @@
  4074. config ATMEL_TCLIB
  4075. bool "Atmel AT32/AT91 Timer/Counter Library"
  4076. depends on (AVR32 || ARCH_AT91)
  4077. + default y if PREEMPT_RT_FULL
  4078. help
  4079. Select this if you want a library to allocate the Timer/Counter
  4080. blocks found on many Atmel processors. This facilitates using
  4081. @@ -77,8 +78,7 @@
  4082. are combined to make a single 32-bit timer.
  4083. When GENERIC_CLOCKEVENTS is defined, the third timer channel
  4084. - may be used as a clock event device supporting oneshot mode
  4085. - (delays of up to two seconds) based on the 32 KiHz clock.
  4086. + may be used as a clock event device supporting oneshot mode.
  4087. config ATMEL_TCB_CLKSRC_BLOCK
  4088. int
  4089. @@ -92,6 +92,15 @@
  4090. TC can be used for other purposes, such as PWM generation and
  4091. interval timing.
  4092. +config ATMEL_TCB_CLKSRC_USE_SLOW_CLOCK
  4093. + bool "TC Block use 32 KiHz clock"
  4094. + depends on ATMEL_TCB_CLKSRC
  4095. + default y if !PREEMPT_RT_FULL
  4096. + help
  4097. + Select this to use 32 KiHz base clock rate as TC block clock
  4098. + source for clock events.
  4099. +
  4100. +
  4101. config DUMMY_IRQ
  4102. tristate "Dummy IRQ handler"
  4103. default n
  4104. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/mmc/host/mmci.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/mmc/host/mmci.c
  4105. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/mmc/host/mmci.c 2017-04-16 10:37:53.000000000 +0200
  4106. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/mmc/host/mmci.c 2017-04-18 17:54:23.000000000 +0200
  4107. @@ -1147,15 +1147,12 @@
  4108. struct sg_mapping_iter *sg_miter = &host->sg_miter;
  4109. struct variant_data *variant = host->variant;
  4110. void __iomem *base = host->base;
  4111. - unsigned long flags;
  4112. u32 status;
  4113. status = readl(base + MMCISTATUS);
  4114. dev_dbg(mmc_dev(host->mmc), "irq1 (pio) %08x\n", status);
  4115. - local_irq_save(flags);
  4116. -
  4117. do {
  4118. unsigned int remain, len;
  4119. char *buffer;
  4120. @@ -1195,8 +1192,6 @@
  4121. sg_miter_stop(sg_miter);
  4122. - local_irq_restore(flags);
  4123. -
  4124. /*
  4125. * If we have less than the fifo 'half-full' threshold to transfer,
  4126. * trigger a PIO interrupt as soon as any data is available.
  4127. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/net/ethernet/3com/3c59x.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/net/ethernet/3com/3c59x.c
  4128. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/net/ethernet/3com/3c59x.c 2017-04-16 10:37:55.000000000 +0200
  4129. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/net/ethernet/3com/3c59x.c 2017-04-18 17:54:23.000000000 +0200
  4130. @@ -842,9 +842,9 @@
  4131. {
  4132. struct vortex_private *vp = netdev_priv(dev);
  4133. unsigned long flags;
  4134. - local_irq_save(flags);
  4135. + local_irq_save_nort(flags);
  4136. (vp->full_bus_master_rx ? boomerang_interrupt:vortex_interrupt)(dev->irq,dev);
  4137. - local_irq_restore(flags);
  4138. + local_irq_restore_nort(flags);
  4139. }
  4140. #endif
  4141. @@ -1910,12 +1910,12 @@
  4142. * Block interrupts because vortex_interrupt does a bare spin_lock()
  4143. */
  4144. unsigned long flags;
  4145. - local_irq_save(flags);
  4146. + local_irq_save_nort(flags);
  4147. if (vp->full_bus_master_tx)
  4148. boomerang_interrupt(dev->irq, dev);
  4149. else
  4150. vortex_interrupt(dev->irq, dev);
  4151. - local_irq_restore(flags);
  4152. + local_irq_restore_nort(flags);
  4153. }
  4154. }
  4155. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/net/ethernet/realtek/8139too.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/net/ethernet/realtek/8139too.c
  4156. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/net/ethernet/realtek/8139too.c 2017-04-16 10:37:58.000000000 +0200
  4157. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/net/ethernet/realtek/8139too.c 2017-04-18 17:54:23.000000000 +0200
  4158. @@ -2233,7 +2233,7 @@
  4159. struct rtl8139_private *tp = netdev_priv(dev);
  4160. const int irq = tp->pci_dev->irq;
  4161. - disable_irq(irq);
  4162. + disable_irq_nosync(irq);
  4163. rtl8139_interrupt(irq, dev);
  4164. enable_irq(irq);
  4165. }
  4166. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/net/wireless/intersil/orinoco/orinoco_usb.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/net/wireless/intersil/orinoco/orinoco_usb.c
  4167. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/net/wireless/intersil/orinoco/orinoco_usb.c 2017-04-16 10:38:02.000000000 +0200
  4168. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/net/wireless/intersil/orinoco/orinoco_usb.c 2017-04-18 17:54:24.000000000 +0200
  4169. @@ -697,7 +697,7 @@
  4170. while (!ctx->done.done && msecs--)
  4171. udelay(1000);
  4172. } else {
  4173. - wait_event_interruptible(ctx->done.wait,
  4174. + swait_event_interruptible(ctx->done.wait,
  4175. ctx->done.done);
  4176. }
  4177. break;
  4178. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/pci/access.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/pci/access.c
  4179. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/pci/access.c 2017-04-16 10:38:05.000000000 +0200
  4180. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/pci/access.c 2017-04-18 17:54:24.000000000 +0200
  4181. @@ -672,7 +672,7 @@
  4182. WARN_ON(!dev->block_cfg_access);
  4183. dev->block_cfg_access = 0;
  4184. - wake_up_all(&pci_cfg_wait);
  4185. + wake_up_all_locked(&pci_cfg_wait);
  4186. raw_spin_unlock_irqrestore(&pci_lock, flags);
  4187. }
  4188. EXPORT_SYMBOL_GPL(pci_cfg_access_unlock);
  4189. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/pinctrl/qcom/pinctrl-msm.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/pinctrl/qcom/pinctrl-msm.c
  4190. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/pinctrl/qcom/pinctrl-msm.c 2017-04-16 10:38:05.000000000 +0200
  4191. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/pinctrl/qcom/pinctrl-msm.c 2017-04-18 17:54:24.000000000 +0200
  4192. @@ -61,7 +61,7 @@
  4193. struct notifier_block restart_nb;
  4194. int irq;
  4195. - spinlock_t lock;
  4196. + raw_spinlock_t lock;
  4197. DECLARE_BITMAP(dual_edge_irqs, MAX_NR_GPIO);
  4198. DECLARE_BITMAP(enabled_irqs, MAX_NR_GPIO);
  4199. @@ -153,14 +153,14 @@
  4200. if (WARN_ON(i == g->nfuncs))
  4201. return -EINVAL;
  4202. - spin_lock_irqsave(&pctrl->lock, flags);
  4203. + raw_spin_lock_irqsave(&pctrl->lock, flags);
  4204. val = readl(pctrl->regs + g->ctl_reg);
  4205. val &= ~mask;
  4206. val |= i << g->mux_bit;
  4207. writel(val, pctrl->regs + g->ctl_reg);
  4208. - spin_unlock_irqrestore(&pctrl->lock, flags);
  4209. + raw_spin_unlock_irqrestore(&pctrl->lock, flags);
  4210. return 0;
  4211. }
  4212. @@ -323,14 +323,14 @@
  4213. break;
  4214. case PIN_CONFIG_OUTPUT:
  4215. /* set output value */
  4216. - spin_lock_irqsave(&pctrl->lock, flags);
  4217. + raw_spin_lock_irqsave(&pctrl->lock, flags);
  4218. val = readl(pctrl->regs + g->io_reg);
  4219. if (arg)
  4220. val |= BIT(g->out_bit);
  4221. else
  4222. val &= ~BIT(g->out_bit);
  4223. writel(val, pctrl->regs + g->io_reg);
  4224. - spin_unlock_irqrestore(&pctrl->lock, flags);
  4225. + raw_spin_unlock_irqrestore(&pctrl->lock, flags);
  4226. /* enable output */
  4227. arg = 1;
  4228. @@ -351,12 +351,12 @@
  4229. return -EINVAL;
  4230. }
  4231. - spin_lock_irqsave(&pctrl->lock, flags);
  4232. + raw_spin_lock_irqsave(&pctrl->lock, flags);
  4233. val = readl(pctrl->regs + g->ctl_reg);
  4234. val &= ~(mask << bit);
  4235. val |= arg << bit;
  4236. writel(val, pctrl->regs + g->ctl_reg);
  4237. - spin_unlock_irqrestore(&pctrl->lock, flags);
  4238. + raw_spin_unlock_irqrestore(&pctrl->lock, flags);
  4239. }
  4240. return 0;
  4241. @@ -384,13 +384,13 @@
  4242. g = &pctrl->soc->groups[offset];
  4243. - spin_lock_irqsave(&pctrl->lock, flags);
  4244. + raw_spin_lock_irqsave(&pctrl->lock, flags);
  4245. val = readl(pctrl->regs + g->ctl_reg);
  4246. val &= ~BIT(g->oe_bit);
  4247. writel(val, pctrl->regs + g->ctl_reg);
  4248. - spin_unlock_irqrestore(&pctrl->lock, flags);
  4249. + raw_spin_unlock_irqrestore(&pctrl->lock, flags);
  4250. return 0;
  4251. }
  4252. @@ -404,7 +404,7 @@
  4253. g = &pctrl->soc->groups[offset];
  4254. - spin_lock_irqsave(&pctrl->lock, flags);
  4255. + raw_spin_lock_irqsave(&pctrl->lock, flags);
  4256. val = readl(pctrl->regs + g->io_reg);
  4257. if (value)
  4258. @@ -417,7 +417,7 @@
  4259. val |= BIT(g->oe_bit);
  4260. writel(val, pctrl->regs + g->ctl_reg);
  4261. - spin_unlock_irqrestore(&pctrl->lock, flags);
  4262. + raw_spin_unlock_irqrestore(&pctrl->lock, flags);
  4263. return 0;
  4264. }
  4265. @@ -443,7 +443,7 @@
  4266. g = &pctrl->soc->groups[offset];
  4267. - spin_lock_irqsave(&pctrl->lock, flags);
  4268. + raw_spin_lock_irqsave(&pctrl->lock, flags);
  4269. val = readl(pctrl->regs + g->io_reg);
  4270. if (value)
  4271. @@ -452,7 +452,7 @@
  4272. val &= ~BIT(g->out_bit);
  4273. writel(val, pctrl->regs + g->io_reg);
  4274. - spin_unlock_irqrestore(&pctrl->lock, flags);
  4275. + raw_spin_unlock_irqrestore(&pctrl->lock, flags);
  4276. }
  4277. #ifdef CONFIG_DEBUG_FS
  4278. @@ -571,7 +571,7 @@
  4279. g = &pctrl->soc->groups[d->hwirq];
  4280. - spin_lock_irqsave(&pctrl->lock, flags);
  4281. + raw_spin_lock_irqsave(&pctrl->lock, flags);
  4282. val = readl(pctrl->regs + g->intr_cfg_reg);
  4283. val &= ~BIT(g->intr_enable_bit);
  4284. @@ -579,7 +579,7 @@
  4285. clear_bit(d->hwirq, pctrl->enabled_irqs);
  4286. - spin_unlock_irqrestore(&pctrl->lock, flags);
  4287. + raw_spin_unlock_irqrestore(&pctrl->lock, flags);
  4288. }
  4289. static void msm_gpio_irq_unmask(struct irq_data *d)
  4290. @@ -592,7 +592,7 @@
  4291. g = &pctrl->soc->groups[d->hwirq];
  4292. - spin_lock_irqsave(&pctrl->lock, flags);
  4293. + raw_spin_lock_irqsave(&pctrl->lock, flags);
  4294. val = readl(pctrl->regs + g->intr_cfg_reg);
  4295. val |= BIT(g->intr_enable_bit);
  4296. @@ -600,7 +600,7 @@
  4297. set_bit(d->hwirq, pctrl->enabled_irqs);
  4298. - spin_unlock_irqrestore(&pctrl->lock, flags);
  4299. + raw_spin_unlock_irqrestore(&pctrl->lock, flags);
  4300. }
  4301. static void msm_gpio_irq_ack(struct irq_data *d)
  4302. @@ -613,7 +613,7 @@
  4303. g = &pctrl->soc->groups[d->hwirq];
  4304. - spin_lock_irqsave(&pctrl->lock, flags);
  4305. + raw_spin_lock_irqsave(&pctrl->lock, flags);
  4306. val = readl(pctrl->regs + g->intr_status_reg);
  4307. if (g->intr_ack_high)
  4308. @@ -625,7 +625,7 @@
  4309. if (test_bit(d->hwirq, pctrl->dual_edge_irqs))
  4310. msm_gpio_update_dual_edge_pos(pctrl, g, d);
  4311. - spin_unlock_irqrestore(&pctrl->lock, flags);
  4312. + raw_spin_unlock_irqrestore(&pctrl->lock, flags);
  4313. }
  4314. static int msm_gpio_irq_set_type(struct irq_data *d, unsigned int type)
  4315. @@ -638,7 +638,7 @@
  4316. g = &pctrl->soc->groups[d->hwirq];
  4317. - spin_lock_irqsave(&pctrl->lock, flags);
  4318. + raw_spin_lock_irqsave(&pctrl->lock, flags);
  4319. /*
  4320. * For hw without possibility of detecting both edges
  4321. @@ -712,7 +712,7 @@
  4322. if (test_bit(d->hwirq, pctrl->dual_edge_irqs))
  4323. msm_gpio_update_dual_edge_pos(pctrl, g, d);
  4324. - spin_unlock_irqrestore(&pctrl->lock, flags);
  4325. + raw_spin_unlock_irqrestore(&pctrl->lock, flags);
  4326. if (type & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH))
  4327. irq_set_handler_locked(d, handle_level_irq);
  4328. @@ -728,11 +728,11 @@
  4329. struct msm_pinctrl *pctrl = gpiochip_get_data(gc);
  4330. unsigned long flags;
  4331. - spin_lock_irqsave(&pctrl->lock, flags);
  4332. + raw_spin_lock_irqsave(&pctrl->lock, flags);
  4333. irq_set_irq_wake(pctrl->irq, on);
  4334. - spin_unlock_irqrestore(&pctrl->lock, flags);
  4335. + raw_spin_unlock_irqrestore(&pctrl->lock, flags);
  4336. return 0;
  4337. }
  4338. @@ -878,7 +878,7 @@
  4339. pctrl->soc = soc_data;
  4340. pctrl->chip = msm_gpio_template;
  4341. - spin_lock_init(&pctrl->lock);
  4342. + raw_spin_lock_init(&pctrl->lock);
  4343. res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  4344. pctrl->regs = devm_ioremap_resource(&pdev->dev, res);
  4345. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/scsi/fcoe/fcoe.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/scsi/fcoe/fcoe.c
  4346. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/scsi/fcoe/fcoe.c 2017-04-16 10:38:09.000000000 +0200
  4347. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/scsi/fcoe/fcoe.c 2017-04-18 17:54:24.000000000 +0200
  4348. @@ -1455,11 +1455,11 @@
  4349. static int fcoe_alloc_paged_crc_eof(struct sk_buff *skb, int tlen)
  4350. {
  4351. struct fcoe_percpu_s *fps;
  4352. - int rc;
  4353. + int rc, cpu = get_cpu_light();
  4354. - fps = &get_cpu_var(fcoe_percpu);
  4355. + fps = &per_cpu(fcoe_percpu, cpu);
  4356. rc = fcoe_get_paged_crc_eof(skb, tlen, fps);
  4357. - put_cpu_var(fcoe_percpu);
  4358. + put_cpu_light();
  4359. return rc;
  4360. }
  4361. @@ -1646,11 +1646,11 @@
  4362. return 0;
  4363. }
  4364. - stats = per_cpu_ptr(lport->stats, get_cpu());
  4365. + stats = per_cpu_ptr(lport->stats, get_cpu_light());
  4366. stats->InvalidCRCCount++;
  4367. if (stats->InvalidCRCCount < 5)
  4368. printk(KERN_WARNING "fcoe: dropping frame with CRC error\n");
  4369. - put_cpu();
  4370. + put_cpu_light();
  4371. return -EINVAL;
  4372. }
  4373. @@ -1693,7 +1693,7 @@
  4374. */
  4375. hp = (struct fcoe_hdr *) skb_network_header(skb);
  4376. - stats = per_cpu_ptr(lport->stats, get_cpu());
  4377. + stats = per_cpu_ptr(lport->stats, get_cpu_light());
  4378. if (unlikely(FC_FCOE_DECAPS_VER(hp) != FC_FCOE_VER)) {
  4379. if (stats->ErrorFrames < 5)
  4380. printk(KERN_WARNING "fcoe: FCoE version "
  4381. @@ -1725,13 +1725,13 @@
  4382. goto drop;
  4383. if (!fcoe_filter_frames(lport, fp)) {
  4384. - put_cpu();
  4385. + put_cpu_light();
  4386. fc_exch_recv(lport, fp);
  4387. return;
  4388. }
  4389. drop:
  4390. stats->ErrorFrames++;
  4391. - put_cpu();
  4392. + put_cpu_light();
  4393. kfree_skb(skb);
  4394. }
  4395. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/scsi/fcoe/fcoe_ctlr.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/scsi/fcoe/fcoe_ctlr.c
  4396. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/scsi/fcoe/fcoe_ctlr.c 2017-04-16 10:38:09.000000000 +0200
  4397. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/scsi/fcoe/fcoe_ctlr.c 2017-04-18 17:54:24.000000000 +0200
  4398. @@ -834,7 +834,7 @@
  4399. INIT_LIST_HEAD(&del_list);
  4400. - stats = per_cpu_ptr(fip->lp->stats, get_cpu());
  4401. + stats = per_cpu_ptr(fip->lp->stats, get_cpu_light());
  4402. list_for_each_entry_safe(fcf, next, &fip->fcfs, list) {
  4403. deadline = fcf->time + fcf->fka_period + fcf->fka_period / 2;
  4404. @@ -870,7 +870,7 @@
  4405. sel_time = fcf->time;
  4406. }
  4407. }
  4408. - put_cpu();
  4409. + put_cpu_light();
  4410. list_for_each_entry_safe(fcf, next, &del_list, list) {
  4411. /* Removes fcf from current list */
  4412. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/scsi/libfc/fc_exch.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/scsi/libfc/fc_exch.c
  4413. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/scsi/libfc/fc_exch.c 2017-04-16 10:38:09.000000000 +0200
  4414. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/scsi/libfc/fc_exch.c 2017-04-18 17:54:24.000000000 +0200
  4415. @@ -814,10 +814,10 @@
  4416. }
  4417. memset(ep, 0, sizeof(*ep));
  4418. - cpu = get_cpu();
  4419. + cpu = get_cpu_light();
  4420. pool = per_cpu_ptr(mp->pool, cpu);
  4421. spin_lock_bh(&pool->lock);
  4422. - put_cpu();
  4423. + put_cpu_light();
  4424. /* peek cache of free slot */
  4425. if (pool->left != FC_XID_UNKNOWN) {
  4426. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/scsi/libsas/sas_ata.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/scsi/libsas/sas_ata.c
  4427. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/scsi/libsas/sas_ata.c 2017-04-16 10:38:09.000000000 +0200
  4428. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/scsi/libsas/sas_ata.c 2017-04-18 17:54:24.000000000 +0200
  4429. @@ -190,7 +190,7 @@
  4430. /* TODO: audit callers to ensure they are ready for qc_issue to
  4431. * unconditionally re-enable interrupts
  4432. */
  4433. - local_irq_save(flags);
  4434. + local_irq_save_nort(flags);
  4435. spin_unlock(ap->lock);
  4436. /* If the device fell off, no sense in issuing commands */
  4437. @@ -252,7 +252,7 @@
  4438. out:
  4439. spin_lock(ap->lock);
  4440. - local_irq_restore(flags);
  4441. + local_irq_restore_nort(flags);
  4442. return ret;
  4443. }
  4444. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/scsi/qla2xxx/qla_inline.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/scsi/qla2xxx/qla_inline.h
  4445. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/scsi/qla2xxx/qla_inline.h 2017-04-16 10:38:10.000000000 +0200
  4446. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/scsi/qla2xxx/qla_inline.h 2017-04-18 17:54:25.000000000 +0200
  4447. @@ -59,12 +59,12 @@
  4448. {
  4449. unsigned long flags;
  4450. struct qla_hw_data *ha = rsp->hw;
  4451. - local_irq_save(flags);
  4452. + local_irq_save_nort(flags);
  4453. if (IS_P3P_TYPE(ha))
  4454. qla82xx_poll(0, rsp);
  4455. else
  4456. ha->isp_ops->intr_handler(0, rsp);
  4457. - local_irq_restore(flags);
  4458. + local_irq_restore_nort(flags);
  4459. }
  4460. static inline uint8_t *
  4461. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/scsi/qla2xxx/qla_isr.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/scsi/qla2xxx/qla_isr.c
  4462. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/scsi/qla2xxx/qla_isr.c 2017-04-16 10:38:10.000000000 +0200
  4463. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/scsi/qla2xxx/qla_isr.c 2017-04-18 17:54:25.000000000 +0200
  4464. @@ -3125,7 +3125,11 @@
  4465. * kref_put().
  4466. */
  4467. kref_get(&qentry->irq_notify.kref);
  4468. +#ifdef CONFIG_PREEMPT_RT_BASE
  4469. + swork_queue(&qentry->irq_notify.swork);
  4470. +#else
  4471. schedule_work(&qentry->irq_notify.work);
  4472. +#endif
  4473. }
  4474. /*
  4475. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/thermal/x86_pkg_temp_thermal.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/thermal/x86_pkg_temp_thermal.c
  4476. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/thermal/x86_pkg_temp_thermal.c 2017-04-16 10:38:14.000000000 +0200
  4477. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/thermal/x86_pkg_temp_thermal.c 2017-04-18 17:54:25.000000000 +0200
  4478. @@ -29,6 +29,7 @@
  4479. #include <linux/pm.h>
  4480. #include <linux/thermal.h>
  4481. #include <linux/debugfs.h>
  4482. +#include <linux/swork.h>
  4483. #include <asm/cpu_device_id.h>
  4484. #include <asm/mce.h>
  4485. @@ -353,7 +354,7 @@
  4486. }
  4487. }
  4488. -static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val)
  4489. +static void platform_thermal_notify_work(struct swork_event *event)
  4490. {
  4491. unsigned long flags;
  4492. int cpu = smp_processor_id();
  4493. @@ -370,7 +371,7 @@
  4494. pkg_work_scheduled[phy_id]) {
  4495. disable_pkg_thres_interrupt();
  4496. spin_unlock_irqrestore(&pkg_work_lock, flags);
  4497. - return -EINVAL;
  4498. + return;
  4499. }
  4500. pkg_work_scheduled[phy_id] = 1;
  4501. spin_unlock_irqrestore(&pkg_work_lock, flags);
  4502. @@ -379,9 +380,48 @@
  4503. schedule_delayed_work_on(cpu,
  4504. &per_cpu(pkg_temp_thermal_threshold_work, cpu),
  4505. msecs_to_jiffies(notify_delay_ms));
  4506. +}
  4507. +
  4508. +#ifdef CONFIG_PREEMPT_RT_FULL
  4509. +static struct swork_event notify_work;
  4510. +
  4511. +static int thermal_notify_work_init(void)
  4512. +{
  4513. + int err;
  4514. +
  4515. + err = swork_get();
  4516. + if (err)
  4517. + return err;
  4518. +
  4519. + INIT_SWORK(&notify_work, platform_thermal_notify_work);
  4520. return 0;
  4521. }
  4522. +static void thermal_notify_work_cleanup(void)
  4523. +{
  4524. + swork_put();
  4525. +}
  4526. +
  4527. +static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val)
  4528. +{
  4529. + swork_queue(&notify_work);
  4530. + return 0;
  4531. +}
  4532. +
  4533. +#else /* !CONFIG_PREEMPT_RT_FULL */
  4534. +
  4535. +static int thermal_notify_work_init(void) { return 0; }
  4536. +
  4537. +static void thermal_notify_work_cleanup(void) { }
  4538. +
  4539. +static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val)
  4540. +{
  4541. + platform_thermal_notify_work(NULL);
  4542. +
  4543. + return 0;
  4544. +}
  4545. +#endif /* CONFIG_PREEMPT_RT_FULL */
  4546. +
  4547. static int find_siblings_cpu(int cpu)
  4548. {
  4549. int i;
  4550. @@ -585,6 +625,9 @@
  4551. if (!x86_match_cpu(pkg_temp_thermal_ids))
  4552. return -ENODEV;
  4553. + if (!thermal_notify_work_init())
  4554. + return -ENODEV;
  4555. +
  4556. spin_lock_init(&pkg_work_lock);
  4557. platform_thermal_package_notify =
  4558. pkg_temp_thermal_platform_thermal_notify;
  4559. @@ -609,7 +652,7 @@
  4560. kfree(pkg_work_scheduled);
  4561. platform_thermal_package_notify = NULL;
  4562. platform_thermal_package_rate_control = NULL;
  4563. -
  4564. + thermal_notify_work_cleanup();
  4565. return -ENODEV;
  4566. }
  4567. @@ -634,6 +677,7 @@
  4568. mutex_unlock(&phy_dev_list_mutex);
  4569. platform_thermal_package_notify = NULL;
  4570. platform_thermal_package_rate_control = NULL;
  4571. + thermal_notify_work_cleanup();
  4572. for_each_online_cpu(i)
  4573. cancel_delayed_work_sync(
  4574. &per_cpu(pkg_temp_thermal_threshold_work, i));
  4575. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/tty/serial/8250/8250_core.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/tty/serial/8250/8250_core.c
  4576. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/tty/serial/8250/8250_core.c 2017-04-16 10:38:14.000000000 +0200
  4577. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/tty/serial/8250/8250_core.c 2017-04-18 17:54:25.000000000 +0200
  4578. @@ -58,7 +58,16 @@
  4579. static unsigned int skip_txen_test; /* force skip of txen test at init time */
  4580. -#define PASS_LIMIT 512
  4581. +/*
  4582. + * On -rt we can have a more delays, and legitimately
  4583. + * so - so don't drop work spuriously and spam the
  4584. + * syslog:
  4585. + */
  4586. +#ifdef CONFIG_PREEMPT_RT_FULL
  4587. +# define PASS_LIMIT 1000000
  4588. +#else
  4589. +# define PASS_LIMIT 512
  4590. +#endif
  4591. #include <asm/serial.h>
  4592. /*
  4593. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/tty/serial/8250/8250_port.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/tty/serial/8250/8250_port.c
  4594. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/tty/serial/8250/8250_port.c 2017-04-16 10:38:14.000000000 +0200
  4595. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/tty/serial/8250/8250_port.c 2017-04-18 17:54:25.000000000 +0200
  4596. @@ -35,6 +35,7 @@
  4597. #include <linux/nmi.h>
  4598. #include <linux/mutex.h>
  4599. #include <linux/slab.h>
  4600. +#include <linux/kdb.h>
  4601. #include <linux/uaccess.h>
  4602. #include <linux/pm_runtime.h>
  4603. #include <linux/timer.h>
  4604. @@ -3144,9 +3145,9 @@
  4605. serial8250_rpm_get(up);
  4606. - if (port->sysrq)
  4607. + if (port->sysrq || oops_in_progress)
  4608. locked = 0;
  4609. - else if (oops_in_progress)
  4610. + else if (in_kdb_printk())
  4611. locked = spin_trylock_irqsave(&port->lock, flags);
  4612. else
  4613. spin_lock_irqsave(&port->lock, flags);
  4614. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/tty/serial/amba-pl011.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/tty/serial/amba-pl011.c
  4615. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/tty/serial/amba-pl011.c 2017-04-16 10:38:14.000000000 +0200
  4616. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/tty/serial/amba-pl011.c 2017-04-18 17:54:25.000000000 +0200
  4617. @@ -2211,13 +2211,19 @@
  4618. clk_enable(uap->clk);
  4619. - local_irq_save(flags);
  4620. + /*
  4621. + * local_irq_save(flags);
  4622. + *
  4623. + * This local_irq_save() is nonsense. If we come in via sysrq
  4624. + * handling then interrupts are already disabled. Aside of
  4625. + * that the port.sysrq check is racy on SMP regardless.
  4626. + */
  4627. if (uap->port.sysrq)
  4628. locked = 0;
  4629. else if (oops_in_progress)
  4630. - locked = spin_trylock(&uap->port.lock);
  4631. + locked = spin_trylock_irqsave(&uap->port.lock, flags);
  4632. else
  4633. - spin_lock(&uap->port.lock);
  4634. + spin_lock_irqsave(&uap->port.lock, flags);
  4635. /*
  4636. * First save the CR then disable the interrupts
  4637. @@ -2241,8 +2247,7 @@
  4638. pl011_write(old_cr, uap, REG_CR);
  4639. if (locked)
  4640. - spin_unlock(&uap->port.lock);
  4641. - local_irq_restore(flags);
  4642. + spin_unlock_irqrestore(&uap->port.lock, flags);
  4643. clk_disable(uap->clk);
  4644. }
  4645. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/tty/serial/omap-serial.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/tty/serial/omap-serial.c
  4646. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/tty/serial/omap-serial.c 2017-04-16 10:38:15.000000000 +0200
  4647. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/tty/serial/omap-serial.c 2017-04-18 17:54:25.000000000 +0200
  4648. @@ -1257,13 +1257,10 @@
  4649. pm_runtime_get_sync(up->dev);
  4650. - local_irq_save(flags);
  4651. - if (up->port.sysrq)
  4652. - locked = 0;
  4653. - else if (oops_in_progress)
  4654. - locked = spin_trylock(&up->port.lock);
  4655. + if (up->port.sysrq || oops_in_progress)
  4656. + locked = spin_trylock_irqsave(&up->port.lock, flags);
  4657. else
  4658. - spin_lock(&up->port.lock);
  4659. + spin_lock_irqsave(&up->port.lock, flags);
  4660. /*
  4661. * First save the IER then disable the interrupts
  4662. @@ -1292,8 +1289,7 @@
  4663. pm_runtime_mark_last_busy(up->dev);
  4664. pm_runtime_put_autosuspend(up->dev);
  4665. if (locked)
  4666. - spin_unlock(&up->port.lock);
  4667. - local_irq_restore(flags);
  4668. + spin_unlock_irqrestore(&up->port.lock, flags);
  4669. }
  4670. static int __init
  4671. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/usb/core/hcd.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/usb/core/hcd.c
  4672. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/usb/core/hcd.c 2017-04-16 10:38:15.000000000 +0200
  4673. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/usb/core/hcd.c 2017-04-18 17:54:25.000000000 +0200
  4674. @@ -1764,9 +1764,9 @@
  4675. * and no one may trigger the above deadlock situation when
  4676. * running complete() in tasklet.
  4677. */
  4678. - local_irq_save(flags);
  4679. + local_irq_save_nort(flags);
  4680. urb->complete(urb);
  4681. - local_irq_restore(flags);
  4682. + local_irq_restore_nort(flags);
  4683. usb_anchor_resume_wakeups(anchor);
  4684. atomic_dec(&urb->use_count);
  4685. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/usb/gadget/function/f_fs.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/usb/gadget/function/f_fs.c
  4686. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/usb/gadget/function/f_fs.c 2017-04-16 10:38:15.000000000 +0200
  4687. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/usb/gadget/function/f_fs.c 2017-04-18 17:54:25.000000000 +0200
  4688. @@ -1593,7 +1593,7 @@
  4689. pr_info("%s(): freeing\n", __func__);
  4690. ffs_data_clear(ffs);
  4691. BUG_ON(waitqueue_active(&ffs->ev.waitq) ||
  4692. - waitqueue_active(&ffs->ep0req_completion.wait));
  4693. + swait_active(&ffs->ep0req_completion.wait));
  4694. kfree(ffs->dev_name);
  4695. kfree(ffs);
  4696. }
  4697. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/usb/gadget/legacy/inode.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/usb/gadget/legacy/inode.c
  4698. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/drivers/usb/gadget/legacy/inode.c 2017-04-16 10:38:15.000000000 +0200
  4699. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/drivers/usb/gadget/legacy/inode.c 2017-04-18 17:54:25.000000000 +0200
  4700. @@ -346,7 +346,7 @@
  4701. spin_unlock_irq (&epdata->dev->lock);
  4702. if (likely (value == 0)) {
  4703. - value = wait_event_interruptible (done.wait, done.done);
  4704. + value = swait_event_interruptible (done.wait, done.done);
  4705. if (value != 0) {
  4706. spin_lock_irq (&epdata->dev->lock);
  4707. if (likely (epdata->ep != NULL)) {
  4708. @@ -355,7 +355,7 @@
  4709. usb_ep_dequeue (epdata->ep, epdata->req);
  4710. spin_unlock_irq (&epdata->dev->lock);
  4711. - wait_event (done.wait, done.done);
  4712. + swait_event (done.wait, done.done);
  4713. if (epdata->status == -ECONNRESET)
  4714. epdata->status = -EINTR;
  4715. } else {
  4716. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/aio.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/aio.c
  4717. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/aio.c 2017-04-16 10:38:19.000000000 +0200
  4718. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/aio.c 2017-04-18 17:54:25.000000000 +0200
  4719. @@ -40,6 +40,7 @@
  4720. #include <linux/ramfs.h>
  4721. #include <linux/percpu-refcount.h>
  4722. #include <linux/mount.h>
  4723. +#include <linux/swork.h>
  4724. #include <asm/kmap_types.h>
  4725. #include <asm/uaccess.h>
  4726. @@ -115,7 +116,7 @@
  4727. struct page **ring_pages;
  4728. long nr_pages;
  4729. - struct work_struct free_work;
  4730. + struct swork_event free_work;
  4731. /*
  4732. * signals when all in-flight requests are done
  4733. @@ -258,6 +259,7 @@
  4734. .mount = aio_mount,
  4735. .kill_sb = kill_anon_super,
  4736. };
  4737. + BUG_ON(swork_get());
  4738. aio_mnt = kern_mount(&aio_fs);
  4739. if (IS_ERR(aio_mnt))
  4740. panic("Failed to create aio fs mount.");
  4741. @@ -581,9 +583,9 @@
  4742. return cancel(&kiocb->common);
  4743. }
  4744. -static void free_ioctx(struct work_struct *work)
  4745. +static void free_ioctx(struct swork_event *sev)
  4746. {
  4747. - struct kioctx *ctx = container_of(work, struct kioctx, free_work);
  4748. + struct kioctx *ctx = container_of(sev, struct kioctx, free_work);
  4749. pr_debug("freeing %p\n", ctx);
  4750. @@ -602,8 +604,8 @@
  4751. if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count))
  4752. complete(&ctx->rq_wait->comp);
  4753. - INIT_WORK(&ctx->free_work, free_ioctx);
  4754. - schedule_work(&ctx->free_work);
  4755. + INIT_SWORK(&ctx->free_work, free_ioctx);
  4756. + swork_queue(&ctx->free_work);
  4757. }
  4758. /*
  4759. @@ -611,9 +613,9 @@
  4760. * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
  4761. * now it's safe to cancel any that need to be.
  4762. */
  4763. -static void free_ioctx_users(struct percpu_ref *ref)
  4764. +static void free_ioctx_users_work(struct swork_event *sev)
  4765. {
  4766. - struct kioctx *ctx = container_of(ref, struct kioctx, users);
  4767. + struct kioctx *ctx = container_of(sev, struct kioctx, free_work);
  4768. struct aio_kiocb *req;
  4769. spin_lock_irq(&ctx->ctx_lock);
  4770. @@ -632,6 +634,14 @@
  4771. percpu_ref_put(&ctx->reqs);
  4772. }
  4773. +static void free_ioctx_users(struct percpu_ref *ref)
  4774. +{
  4775. + struct kioctx *ctx = container_of(ref, struct kioctx, users);
  4776. +
  4777. + INIT_SWORK(&ctx->free_work, free_ioctx_users_work);
  4778. + swork_queue(&ctx->free_work);
  4779. +}
  4780. +
  4781. static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
  4782. {
  4783. unsigned i, new_nr;
  4784. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/autofs4/autofs_i.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/autofs4/autofs_i.h
  4785. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/autofs4/autofs_i.h 2017-04-16 10:38:19.000000000 +0200
  4786. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/autofs4/autofs_i.h 2017-04-18 17:54:25.000000000 +0200
  4787. @@ -31,6 +31,7 @@
  4788. #include <linux/sched.h>
  4789. #include <linux/mount.h>
  4790. #include <linux/namei.h>
  4791. +#include <linux/delay.h>
  4792. #include <asm/current.h>
  4793. #include <linux/uaccess.h>
  4794. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/autofs4/expire.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/autofs4/expire.c
  4795. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/autofs4/expire.c 2017-04-16 10:38:19.000000000 +0200
  4796. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/autofs4/expire.c 2017-04-18 17:54:25.000000000 +0200
  4797. @@ -148,7 +148,7 @@
  4798. parent = p->d_parent;
  4799. if (!spin_trylock(&parent->d_lock)) {
  4800. spin_unlock(&p->d_lock);
  4801. - cpu_relax();
  4802. + cpu_chill();
  4803. goto relock;
  4804. }
  4805. spin_unlock(&p->d_lock);
  4806. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/buffer.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/buffer.c
  4807. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/buffer.c 2017-04-16 10:38:19.000000000 +0200
  4808. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/buffer.c 2017-04-18 17:54:25.000000000 +0200
  4809. @@ -301,8 +301,7 @@
  4810. * decide that the page is now completely done.
  4811. */
  4812. first = page_buffers(page);
  4813. - local_irq_save(flags);
  4814. - bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
  4815. + flags = bh_uptodate_lock_irqsave(first);
  4816. clear_buffer_async_read(bh);
  4817. unlock_buffer(bh);
  4818. tmp = bh;
  4819. @@ -315,8 +314,7 @@
  4820. }
  4821. tmp = tmp->b_this_page;
  4822. } while (tmp != bh);
  4823. - bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  4824. - local_irq_restore(flags);
  4825. + bh_uptodate_unlock_irqrestore(first, flags);
  4826. /*
  4827. * If none of the buffers had errors and they are all
  4828. @@ -328,9 +326,7 @@
  4829. return;
  4830. still_busy:
  4831. - bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  4832. - local_irq_restore(flags);
  4833. - return;
  4834. + bh_uptodate_unlock_irqrestore(first, flags);
  4835. }
  4836. /*
  4837. @@ -358,8 +354,7 @@
  4838. }
  4839. first = page_buffers(page);
  4840. - local_irq_save(flags);
  4841. - bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
  4842. + flags = bh_uptodate_lock_irqsave(first);
  4843. clear_buffer_async_write(bh);
  4844. unlock_buffer(bh);
  4845. @@ -371,15 +366,12 @@
  4846. }
  4847. tmp = tmp->b_this_page;
  4848. }
  4849. - bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  4850. - local_irq_restore(flags);
  4851. + bh_uptodate_unlock_irqrestore(first, flags);
  4852. end_page_writeback(page);
  4853. return;
  4854. still_busy:
  4855. - bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  4856. - local_irq_restore(flags);
  4857. - return;
  4858. + bh_uptodate_unlock_irqrestore(first, flags);
  4859. }
  4860. EXPORT_SYMBOL(end_buffer_async_write);
  4861. @@ -3383,6 +3375,7 @@
  4862. struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
  4863. if (ret) {
  4864. INIT_LIST_HEAD(&ret->b_assoc_buffers);
  4865. + buffer_head_init_locks(ret);
  4866. preempt_disable();
  4867. __this_cpu_inc(bh_accounting.nr);
  4868. recalc_bh_state();
  4869. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/cifs/readdir.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/cifs/readdir.c
  4870. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/cifs/readdir.c 2017-04-16 10:38:20.000000000 +0200
  4871. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/cifs/readdir.c 2017-04-18 17:54:25.000000000 +0200
  4872. @@ -80,7 +80,7 @@
  4873. struct inode *inode;
  4874. struct super_block *sb = parent->d_sb;
  4875. struct cifs_sb_info *cifs_sb = CIFS_SB(sb);
  4876. - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
  4877. + DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(wq);
  4878. cifs_dbg(FYI, "%s: for %s\n", __func__, name->name);
  4879. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/dcache.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/dcache.c
  4880. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/dcache.c 2017-04-16 10:38:20.000000000 +0200
  4881. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/dcache.c 2017-04-18 17:54:25.000000000 +0200
  4882. @@ -19,6 +19,7 @@
  4883. #include <linux/mm.h>
  4884. #include <linux/fs.h>
  4885. #include <linux/fsnotify.h>
  4886. +#include <linux/delay.h>
  4887. #include <linux/slab.h>
  4888. #include <linux/init.h>
  4889. #include <linux/hash.h>
  4890. @@ -750,6 +751,8 @@
  4891. */
  4892. void dput(struct dentry *dentry)
  4893. {
  4894. + struct dentry *parent;
  4895. +
  4896. if (unlikely(!dentry))
  4897. return;
  4898. @@ -788,9 +791,18 @@
  4899. return;
  4900. kill_it:
  4901. - dentry = dentry_kill(dentry);
  4902. - if (dentry) {
  4903. - cond_resched();
  4904. + parent = dentry_kill(dentry);
  4905. + if (parent) {
  4906. + int r;
  4907. +
  4908. + if (parent == dentry) {
  4909. + /* the task with the highest priority won't schedule */
  4910. + r = cond_resched();
  4911. + if (!r)
  4912. + cpu_chill();
  4913. + } else {
  4914. + dentry = parent;
  4915. + }
  4916. goto repeat;
  4917. }
  4918. }
  4919. @@ -2324,7 +2336,7 @@
  4920. if (dentry->d_lockref.count == 1) {
  4921. if (!spin_trylock(&inode->i_lock)) {
  4922. spin_unlock(&dentry->d_lock);
  4923. - cpu_relax();
  4924. + cpu_chill();
  4925. goto again;
  4926. }
  4927. dentry->d_flags &= ~DCACHE_CANT_MOUNT;
  4928. @@ -2384,21 +2396,24 @@
  4929. static void d_wait_lookup(struct dentry *dentry)
  4930. {
  4931. - if (d_in_lookup(dentry)) {
  4932. - DECLARE_WAITQUEUE(wait, current);
  4933. - add_wait_queue(dentry->d_wait, &wait);
  4934. - do {
  4935. - set_current_state(TASK_UNINTERRUPTIBLE);
  4936. - spin_unlock(&dentry->d_lock);
  4937. - schedule();
  4938. - spin_lock(&dentry->d_lock);
  4939. - } while (d_in_lookup(dentry));
  4940. - }
  4941. + struct swait_queue __wait;
  4942. +
  4943. + if (!d_in_lookup(dentry))
  4944. + return;
  4945. +
  4946. + INIT_LIST_HEAD(&__wait.task_list);
  4947. + do {
  4948. + prepare_to_swait(dentry->d_wait, &__wait, TASK_UNINTERRUPTIBLE);
  4949. + spin_unlock(&dentry->d_lock);
  4950. + schedule();
  4951. + spin_lock(&dentry->d_lock);
  4952. + } while (d_in_lookup(dentry));
  4953. + finish_swait(dentry->d_wait, &__wait);
  4954. }
  4955. struct dentry *d_alloc_parallel(struct dentry *parent,
  4956. const struct qstr *name,
  4957. - wait_queue_head_t *wq)
  4958. + struct swait_queue_head *wq)
  4959. {
  4960. unsigned int hash = name->hash;
  4961. struct hlist_bl_head *b = in_lookup_hash(parent, hash);
  4962. @@ -2507,7 +2522,7 @@
  4963. hlist_bl_lock(b);
  4964. dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
  4965. __hlist_bl_del(&dentry->d_u.d_in_lookup_hash);
  4966. - wake_up_all(dentry->d_wait);
  4967. + swake_up_all(dentry->d_wait);
  4968. dentry->d_wait = NULL;
  4969. hlist_bl_unlock(b);
  4970. INIT_HLIST_NODE(&dentry->d_u.d_alias);
  4971. @@ -3604,6 +3619,11 @@
  4972. void __init vfs_caches_init_early(void)
  4973. {
  4974. + int i;
  4975. +
  4976. + for (i = 0; i < ARRAY_SIZE(in_lookup_hashtable); i++)
  4977. + INIT_HLIST_BL_HEAD(&in_lookup_hashtable[i]);
  4978. +
  4979. dcache_init_early();
  4980. inode_init_early();
  4981. }
  4982. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/eventpoll.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/eventpoll.c
  4983. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/eventpoll.c 2017-04-16 10:38:20.000000000 +0200
  4984. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/eventpoll.c 2017-04-18 17:54:25.000000000 +0200
  4985. @@ -510,12 +510,12 @@
  4986. */
  4987. static void ep_poll_safewake(wait_queue_head_t *wq)
  4988. {
  4989. - int this_cpu = get_cpu();
  4990. + int this_cpu = get_cpu_light();
  4991. ep_call_nested(&poll_safewake_ncalls, EP_MAX_NESTS,
  4992. ep_poll_wakeup_proc, NULL, wq, (void *) (long) this_cpu);
  4993. - put_cpu();
  4994. + put_cpu_light();
  4995. }
  4996. static void ep_remove_wait_queue(struct eppoll_entry *pwq)
  4997. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/exec.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/exec.c
  4998. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/exec.c 2017-04-16 10:38:20.000000000 +0200
  4999. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/exec.c 2017-04-18 17:54:25.000000000 +0200
  5000. @@ -1017,12 +1017,14 @@
  5001. }
  5002. }
  5003. task_lock(tsk);
  5004. + preempt_disable_rt();
  5005. active_mm = tsk->active_mm;
  5006. tsk->mm = mm;
  5007. tsk->active_mm = mm;
  5008. activate_mm(active_mm, mm);
  5009. tsk->mm->vmacache_seqnum = 0;
  5010. vmacache_flush(tsk);
  5011. + preempt_enable_rt();
  5012. task_unlock(tsk);
  5013. if (old_mm) {
  5014. up_read(&old_mm->mmap_sem);
  5015. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/fuse/dir.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/fuse/dir.c
  5016. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/fuse/dir.c 2017-04-16 10:38:20.000000000 +0200
  5017. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/fuse/dir.c 2017-04-18 17:54:25.000000000 +0200
  5018. @@ -1191,7 +1191,7 @@
  5019. struct inode *dir = d_inode(parent);
  5020. struct fuse_conn *fc;
  5021. struct inode *inode;
  5022. - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
  5023. + DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(wq);
  5024. if (!o->nodeid) {
  5025. /*
  5026. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/jbd2/checkpoint.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/jbd2/checkpoint.c
  5027. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/jbd2/checkpoint.c 2017-04-16 10:38:20.000000000 +0200
  5028. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/jbd2/checkpoint.c 2017-04-18 17:54:25.000000000 +0200
  5029. @@ -116,6 +116,8 @@
  5030. nblocks = jbd2_space_needed(journal);
  5031. while (jbd2_log_space_left(journal) < nblocks) {
  5032. write_unlock(&journal->j_state_lock);
  5033. + if (current->plug)
  5034. + io_schedule();
  5035. mutex_lock(&journal->j_checkpoint_mutex);
  5036. /*
  5037. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/locks.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/locks.c
  5038. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/locks.c 2017-04-16 10:38:21.000000000 +0200
  5039. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/locks.c 2017-04-18 17:54:25.000000000 +0200
  5040. @@ -935,7 +935,7 @@
  5041. return -ENOMEM;
  5042. }
  5043. - percpu_down_read_preempt_disable(&file_rwsem);
  5044. + percpu_down_read(&file_rwsem);
  5045. spin_lock(&ctx->flc_lock);
  5046. if (request->fl_flags & FL_ACCESS)
  5047. goto find_conflict;
  5048. @@ -976,7 +976,7 @@
  5049. out:
  5050. spin_unlock(&ctx->flc_lock);
  5051. - percpu_up_read_preempt_enable(&file_rwsem);
  5052. + percpu_up_read(&file_rwsem);
  5053. if (new_fl)
  5054. locks_free_lock(new_fl);
  5055. locks_dispose_list(&dispose);
  5056. @@ -1013,7 +1013,7 @@
  5057. new_fl2 = locks_alloc_lock();
  5058. }
  5059. - percpu_down_read_preempt_disable(&file_rwsem);
  5060. + percpu_down_read(&file_rwsem);
  5061. spin_lock(&ctx->flc_lock);
  5062. /*
  5063. * New lock request. Walk all POSIX locks and look for conflicts. If
  5064. @@ -1185,7 +1185,7 @@
  5065. }
  5066. out:
  5067. spin_unlock(&ctx->flc_lock);
  5068. - percpu_up_read_preempt_enable(&file_rwsem);
  5069. + percpu_up_read(&file_rwsem);
  5070. /*
  5071. * Free any unused locks.
  5072. */
  5073. @@ -1460,7 +1460,7 @@
  5074. return error;
  5075. }
  5076. - percpu_down_read_preempt_disable(&file_rwsem);
  5077. + percpu_down_read(&file_rwsem);
  5078. spin_lock(&ctx->flc_lock);
  5079. time_out_leases(inode, &dispose);
  5080. @@ -1512,13 +1512,13 @@
  5081. locks_insert_block(fl, new_fl);
  5082. trace_break_lease_block(inode, new_fl);
  5083. spin_unlock(&ctx->flc_lock);
  5084. - percpu_up_read_preempt_enable(&file_rwsem);
  5085. + percpu_up_read(&file_rwsem);
  5086. locks_dispose_list(&dispose);
  5087. error = wait_event_interruptible_timeout(new_fl->fl_wait,
  5088. !new_fl->fl_next, break_time);
  5089. - percpu_down_read_preempt_disable(&file_rwsem);
  5090. + percpu_down_read(&file_rwsem);
  5091. spin_lock(&ctx->flc_lock);
  5092. trace_break_lease_unblock(inode, new_fl);
  5093. locks_delete_block(new_fl);
  5094. @@ -1535,7 +1535,7 @@
  5095. }
  5096. out:
  5097. spin_unlock(&ctx->flc_lock);
  5098. - percpu_up_read_preempt_enable(&file_rwsem);
  5099. + percpu_up_read(&file_rwsem);
  5100. locks_dispose_list(&dispose);
  5101. locks_free_lock(new_fl);
  5102. return error;
  5103. @@ -1609,7 +1609,7 @@
  5104. ctx = smp_load_acquire(&inode->i_flctx);
  5105. if (ctx && !list_empty_careful(&ctx->flc_lease)) {
  5106. - percpu_down_read_preempt_disable(&file_rwsem);
  5107. + percpu_down_read(&file_rwsem);
  5108. spin_lock(&ctx->flc_lock);
  5109. time_out_leases(inode, &dispose);
  5110. list_for_each_entry(fl, &ctx->flc_lease, fl_list) {
  5111. @@ -1619,7 +1619,7 @@
  5112. break;
  5113. }
  5114. spin_unlock(&ctx->flc_lock);
  5115. - percpu_up_read_preempt_enable(&file_rwsem);
  5116. + percpu_up_read(&file_rwsem);
  5117. locks_dispose_list(&dispose);
  5118. }
  5119. @@ -1694,7 +1694,7 @@
  5120. return -EINVAL;
  5121. }
  5122. - percpu_down_read_preempt_disable(&file_rwsem);
  5123. + percpu_down_read(&file_rwsem);
  5124. spin_lock(&ctx->flc_lock);
  5125. time_out_leases(inode, &dispose);
  5126. error = check_conflicting_open(dentry, arg, lease->fl_flags);
  5127. @@ -1765,7 +1765,7 @@
  5128. lease->fl_lmops->lm_setup(lease, priv);
  5129. out:
  5130. spin_unlock(&ctx->flc_lock);
  5131. - percpu_up_read_preempt_enable(&file_rwsem);
  5132. + percpu_up_read(&file_rwsem);
  5133. locks_dispose_list(&dispose);
  5134. if (is_deleg)
  5135. inode_unlock(inode);
  5136. @@ -1788,7 +1788,7 @@
  5137. return error;
  5138. }
  5139. - percpu_down_read_preempt_disable(&file_rwsem);
  5140. + percpu_down_read(&file_rwsem);
  5141. spin_lock(&ctx->flc_lock);
  5142. list_for_each_entry(fl, &ctx->flc_lease, fl_list) {
  5143. if (fl->fl_file == filp &&
  5144. @@ -1801,7 +1801,7 @@
  5145. if (victim)
  5146. error = fl->fl_lmops->lm_change(victim, F_UNLCK, &dispose);
  5147. spin_unlock(&ctx->flc_lock);
  5148. - percpu_up_read_preempt_enable(&file_rwsem);
  5149. + percpu_up_read(&file_rwsem);
  5150. locks_dispose_list(&dispose);
  5151. return error;
  5152. }
  5153. @@ -2532,13 +2532,13 @@
  5154. if (list_empty(&ctx->flc_lease))
  5155. return;
  5156. - percpu_down_read_preempt_disable(&file_rwsem);
  5157. + percpu_down_read(&file_rwsem);
  5158. spin_lock(&ctx->flc_lock);
  5159. list_for_each_entry_safe(fl, tmp, &ctx->flc_lease, fl_list)
  5160. if (filp == fl->fl_file)
  5161. lease_modify(fl, F_UNLCK, &dispose);
  5162. spin_unlock(&ctx->flc_lock);
  5163. - percpu_up_read_preempt_enable(&file_rwsem);
  5164. + percpu_up_read(&file_rwsem);
  5165. locks_dispose_list(&dispose);
  5166. }
  5167. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/namei.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/namei.c
  5168. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/namei.c 2017-04-16 10:38:21.000000000 +0200
  5169. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/namei.c 2017-04-18 17:54:25.000000000 +0200
  5170. @@ -1626,7 +1626,7 @@
  5171. {
  5172. struct dentry *dentry = ERR_PTR(-ENOENT), *old;
  5173. struct inode *inode = dir->d_inode;
  5174. - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
  5175. + DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(wq);
  5176. inode_lock_shared(inode);
  5177. /* Don't go there if it's already dead */
  5178. @@ -3083,7 +3083,7 @@
  5179. struct dentry *dentry;
  5180. int error, create_error = 0;
  5181. umode_t mode = op->mode;
  5182. - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
  5183. + DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(wq);
  5184. if (unlikely(IS_DEADDIR(dir_inode)))
  5185. return -ENOENT;
  5186. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/namespace.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/namespace.c
  5187. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/namespace.c 2017-04-16 10:38:21.000000000 +0200
  5188. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/namespace.c 2017-04-18 17:54:25.000000000 +0200
  5189. @@ -14,6 +14,7 @@
  5190. #include <linux/mnt_namespace.h>
  5191. #include <linux/user_namespace.h>
  5192. #include <linux/namei.h>
  5193. +#include <linux/delay.h>
  5194. #include <linux/security.h>
  5195. #include <linux/idr.h>
  5196. #include <linux/init.h> /* init_rootfs */
  5197. @@ -356,8 +357,11 @@
  5198. * incremented count after it has set MNT_WRITE_HOLD.
  5199. */
  5200. smp_mb();
  5201. - while (ACCESS_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD)
  5202. - cpu_relax();
  5203. + while (ACCESS_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD) {
  5204. + preempt_enable();
  5205. + cpu_chill();
  5206. + preempt_disable();
  5207. + }
  5208. /*
  5209. * After the slowpath clears MNT_WRITE_HOLD, mnt_is_readonly will
  5210. * be set to match its requirements. So we must not load that until
  5211. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/delegation.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/delegation.c
  5212. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/delegation.c 2017-04-16 10:38:21.000000000 +0200
  5213. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/delegation.c 2017-04-18 17:54:25.000000000 +0200
  5214. @@ -150,11 +150,11 @@
  5215. sp = state->owner;
  5216. /* Block nfs4_proc_unlck */
  5217. mutex_lock(&sp->so_delegreturn_mutex);
  5218. - seq = raw_seqcount_begin(&sp->so_reclaim_seqcount);
  5219. + seq = read_seqbegin(&sp->so_reclaim_seqlock);
  5220. err = nfs4_open_delegation_recall(ctx, state, stateid, type);
  5221. if (!err)
  5222. err = nfs_delegation_claim_locks(ctx, state, stateid);
  5223. - if (!err && read_seqcount_retry(&sp->so_reclaim_seqcount, seq))
  5224. + if (!err && read_seqretry(&sp->so_reclaim_seqlock, seq))
  5225. err = -EAGAIN;
  5226. mutex_unlock(&sp->so_delegreturn_mutex);
  5227. put_nfs_open_context(ctx);
  5228. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/dir.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/dir.c
  5229. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/dir.c 2017-04-16 10:38:21.000000000 +0200
  5230. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/dir.c 2017-04-18 17:54:25.000000000 +0200
  5231. @@ -485,7 +485,7 @@
  5232. void nfs_prime_dcache(struct dentry *parent, struct nfs_entry *entry)
  5233. {
  5234. struct qstr filename = QSTR_INIT(entry->name, entry->len);
  5235. - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
  5236. + DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(wq);
  5237. struct dentry *dentry;
  5238. struct dentry *alias;
  5239. struct inode *dir = d_inode(parent);
  5240. @@ -1487,7 +1487,7 @@
  5241. struct file *file, unsigned open_flags,
  5242. umode_t mode, int *opened)
  5243. {
  5244. - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
  5245. + DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(wq);
  5246. struct nfs_open_context *ctx;
  5247. struct dentry *res;
  5248. struct iattr attr = { .ia_valid = ATTR_OPEN };
  5249. @@ -1802,7 +1802,11 @@
  5250. trace_nfs_rmdir_enter(dir, dentry);
  5251. if (d_really_is_positive(dentry)) {
  5252. +#ifdef CONFIG_PREEMPT_RT_BASE
  5253. + down(&NFS_I(d_inode(dentry))->rmdir_sem);
  5254. +#else
  5255. down_write(&NFS_I(d_inode(dentry))->rmdir_sem);
  5256. +#endif
  5257. error = NFS_PROTO(dir)->rmdir(dir, &dentry->d_name);
  5258. /* Ensure the VFS deletes this inode */
  5259. switch (error) {
  5260. @@ -1812,7 +1816,11 @@
  5261. case -ENOENT:
  5262. nfs_dentry_handle_enoent(dentry);
  5263. }
  5264. +#ifdef CONFIG_PREEMPT_RT_BASE
  5265. + up(&NFS_I(d_inode(dentry))->rmdir_sem);
  5266. +#else
  5267. up_write(&NFS_I(d_inode(dentry))->rmdir_sem);
  5268. +#endif
  5269. } else
  5270. error = NFS_PROTO(dir)->rmdir(dir, &dentry->d_name);
  5271. trace_nfs_rmdir_exit(dir, dentry, error);
  5272. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/inode.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/inode.c
  5273. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/inode.c 2017-04-16 10:38:21.000000000 +0200
  5274. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/inode.c 2017-04-18 17:54:25.000000000 +0200
  5275. @@ -1957,7 +1957,11 @@
  5276. nfsi->nrequests = 0;
  5277. nfsi->commit_info.ncommit = 0;
  5278. atomic_set(&nfsi->commit_info.rpcs_out, 0);
  5279. +#ifdef CONFIG_PREEMPT_RT_BASE
  5280. + sema_init(&nfsi->rmdir_sem, 1);
  5281. +#else
  5282. init_rwsem(&nfsi->rmdir_sem);
  5283. +#endif
  5284. nfs4_init_once(nfsi);
  5285. }
  5286. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/nfs4_fs.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/nfs4_fs.h
  5287. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/nfs4_fs.h 2017-04-16 10:38:21.000000000 +0200
  5288. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/nfs4_fs.h 2017-04-18 17:54:25.000000000 +0200
  5289. @@ -111,7 +111,7 @@
  5290. unsigned long so_flags;
  5291. struct list_head so_states;
  5292. struct nfs_seqid_counter so_seqid;
  5293. - seqcount_t so_reclaim_seqcount;
  5294. + seqlock_t so_reclaim_seqlock;
  5295. struct mutex so_delegreturn_mutex;
  5296. };
  5297. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/nfs4proc.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/nfs4proc.c
  5298. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/nfs4proc.c 2017-04-16 10:38:21.000000000 +0200
  5299. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/nfs4proc.c 2017-04-18 17:54:25.000000000 +0200
  5300. @@ -2695,7 +2695,7 @@
  5301. unsigned int seq;
  5302. int ret;
  5303. - seq = raw_seqcount_begin(&sp->so_reclaim_seqcount);
  5304. + seq = raw_seqcount_begin(&sp->so_reclaim_seqlock.seqcount);
  5305. ret = _nfs4_proc_open(opendata);
  5306. if (ret != 0)
  5307. @@ -2733,7 +2733,7 @@
  5308. if (d_inode(dentry) == state->inode) {
  5309. nfs_inode_attach_open_context(ctx);
  5310. - if (read_seqcount_retry(&sp->so_reclaim_seqcount, seq))
  5311. + if (read_seqretry(&sp->so_reclaim_seqlock, seq))
  5312. nfs4_schedule_stateid_recovery(server, state);
  5313. }
  5314. out:
  5315. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/nfs4state.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/nfs4state.c
  5316. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/nfs4state.c 2017-04-16 10:38:21.000000000 +0200
  5317. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/nfs4state.c 2017-04-18 17:54:25.000000000 +0200
  5318. @@ -488,7 +488,7 @@
  5319. nfs4_init_seqid_counter(&sp->so_seqid);
  5320. atomic_set(&sp->so_count, 1);
  5321. INIT_LIST_HEAD(&sp->so_lru);
  5322. - seqcount_init(&sp->so_reclaim_seqcount);
  5323. + seqlock_init(&sp->so_reclaim_seqlock);
  5324. mutex_init(&sp->so_delegreturn_mutex);
  5325. return sp;
  5326. }
  5327. @@ -1497,8 +1497,12 @@
  5328. * recovering after a network partition or a reboot from a
  5329. * server that doesn't support a grace period.
  5330. */
  5331. +#ifdef CONFIG_PREEMPT_RT_FULL
  5332. + write_seqlock(&sp->so_reclaim_seqlock);
  5333. +#else
  5334. + write_seqcount_begin(&sp->so_reclaim_seqlock.seqcount);
  5335. +#endif
  5336. spin_lock(&sp->so_lock);
  5337. - raw_write_seqcount_begin(&sp->so_reclaim_seqcount);
  5338. restart:
  5339. list_for_each_entry(state, &sp->so_states, open_states) {
  5340. if (!test_and_clear_bit(ops->state_flag_bit, &state->flags))
  5341. @@ -1567,14 +1571,20 @@
  5342. spin_lock(&sp->so_lock);
  5343. goto restart;
  5344. }
  5345. - raw_write_seqcount_end(&sp->so_reclaim_seqcount);
  5346. spin_unlock(&sp->so_lock);
  5347. +#ifdef CONFIG_PREEMPT_RT_FULL
  5348. + write_sequnlock(&sp->so_reclaim_seqlock);
  5349. +#else
  5350. + write_seqcount_end(&sp->so_reclaim_seqlock.seqcount);
  5351. +#endif
  5352. return 0;
  5353. out_err:
  5354. nfs4_put_open_state(state);
  5355. - spin_lock(&sp->so_lock);
  5356. - raw_write_seqcount_end(&sp->so_reclaim_seqcount);
  5357. - spin_unlock(&sp->so_lock);
  5358. +#ifdef CONFIG_PREEMPT_RT_FULL
  5359. + write_sequnlock(&sp->so_reclaim_seqlock);
  5360. +#else
  5361. + write_seqcount_end(&sp->so_reclaim_seqlock.seqcount);
  5362. +#endif
  5363. return status;
  5364. }
  5365. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/unlink.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/unlink.c
  5366. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/nfs/unlink.c 2017-04-16 10:38:21.000000000 +0200
  5367. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/nfs/unlink.c 2017-04-18 17:54:25.000000000 +0200
  5368. @@ -12,7 +12,7 @@
  5369. #include <linux/sunrpc/clnt.h>
  5370. #include <linux/nfs_fs.h>
  5371. #include <linux/sched.h>
  5372. -#include <linux/wait.h>
  5373. +#include <linux/swait.h>
  5374. #include <linux/namei.h>
  5375. #include <linux/fsnotify.h>
  5376. @@ -51,6 +51,29 @@
  5377. rpc_restart_call_prepare(task);
  5378. }
  5379. +#ifdef CONFIG_PREEMPT_RT_BASE
  5380. +static void nfs_down_anon(struct semaphore *sema)
  5381. +{
  5382. + down(sema);
  5383. +}
  5384. +
  5385. +static void nfs_up_anon(struct semaphore *sema)
  5386. +{
  5387. + up(sema);
  5388. +}
  5389. +
  5390. +#else
  5391. +static void nfs_down_anon(struct rw_semaphore *rwsem)
  5392. +{
  5393. + down_read_non_owner(rwsem);
  5394. +}
  5395. +
  5396. +static void nfs_up_anon(struct rw_semaphore *rwsem)
  5397. +{
  5398. + up_read_non_owner(rwsem);
  5399. +}
  5400. +#endif
  5401. +
  5402. /**
  5403. * nfs_async_unlink_release - Release the sillydelete data.
  5404. * @task: rpc_task of the sillydelete
  5405. @@ -64,7 +87,7 @@
  5406. struct dentry *dentry = data->dentry;
  5407. struct super_block *sb = dentry->d_sb;
  5408. - up_read_non_owner(&NFS_I(d_inode(dentry->d_parent))->rmdir_sem);
  5409. + nfs_up_anon(&NFS_I(d_inode(dentry->d_parent))->rmdir_sem);
  5410. d_lookup_done(dentry);
  5411. nfs_free_unlinkdata(data);
  5412. dput(dentry);
  5413. @@ -117,10 +140,10 @@
  5414. struct inode *dir = d_inode(dentry->d_parent);
  5415. struct dentry *alias;
  5416. - down_read_non_owner(&NFS_I(dir)->rmdir_sem);
  5417. + nfs_down_anon(&NFS_I(dir)->rmdir_sem);
  5418. alias = d_alloc_parallel(dentry->d_parent, &data->args.name, &data->wq);
  5419. if (IS_ERR(alias)) {
  5420. - up_read_non_owner(&NFS_I(dir)->rmdir_sem);
  5421. + nfs_up_anon(&NFS_I(dir)->rmdir_sem);
  5422. return 0;
  5423. }
  5424. if (!d_in_lookup(alias)) {
  5425. @@ -142,7 +165,7 @@
  5426. ret = 0;
  5427. spin_unlock(&alias->d_lock);
  5428. dput(alias);
  5429. - up_read_non_owner(&NFS_I(dir)->rmdir_sem);
  5430. + nfs_up_anon(&NFS_I(dir)->rmdir_sem);
  5431. /*
  5432. * If we'd displaced old cached devname, free it. At that
  5433. * point dentry is definitely not a root, so we won't need
  5434. @@ -182,7 +205,7 @@
  5435. goto out_free_name;
  5436. }
  5437. data->res.dir_attr = &data->dir_attr;
  5438. - init_waitqueue_head(&data->wq);
  5439. + init_swait_queue_head(&data->wq);
  5440. status = -EBUSY;
  5441. spin_lock(&dentry->d_lock);
  5442. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/ntfs/aops.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/ntfs/aops.c
  5443. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/ntfs/aops.c 2017-04-16 10:38:21.000000000 +0200
  5444. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/ntfs/aops.c 2017-04-18 17:54:25.000000000 +0200
  5445. @@ -92,13 +92,13 @@
  5446. ofs = 0;
  5447. if (file_ofs < init_size)
  5448. ofs = init_size - file_ofs;
  5449. - local_irq_save(flags);
  5450. + local_irq_save_nort(flags);
  5451. kaddr = kmap_atomic(page);
  5452. memset(kaddr + bh_offset(bh) + ofs, 0,
  5453. bh->b_size - ofs);
  5454. flush_dcache_page(page);
  5455. kunmap_atomic(kaddr);
  5456. - local_irq_restore(flags);
  5457. + local_irq_restore_nort(flags);
  5458. }
  5459. } else {
  5460. clear_buffer_uptodate(bh);
  5461. @@ -107,8 +107,7 @@
  5462. "0x%llx.", (unsigned long long)bh->b_blocknr);
  5463. }
  5464. first = page_buffers(page);
  5465. - local_irq_save(flags);
  5466. - bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
  5467. + flags = bh_uptodate_lock_irqsave(first);
  5468. clear_buffer_async_read(bh);
  5469. unlock_buffer(bh);
  5470. tmp = bh;
  5471. @@ -123,8 +122,7 @@
  5472. }
  5473. tmp = tmp->b_this_page;
  5474. } while (tmp != bh);
  5475. - bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  5476. - local_irq_restore(flags);
  5477. + bh_uptodate_unlock_irqrestore(first, flags);
  5478. /*
  5479. * If none of the buffers had errors then we can set the page uptodate,
  5480. * but we first have to perform the post read mst fixups, if the
  5481. @@ -145,13 +143,13 @@
  5482. recs = PAGE_SIZE / rec_size;
  5483. /* Should have been verified before we got here... */
  5484. BUG_ON(!recs);
  5485. - local_irq_save(flags);
  5486. + local_irq_save_nort(flags);
  5487. kaddr = kmap_atomic(page);
  5488. for (i = 0; i < recs; i++)
  5489. post_read_mst_fixup((NTFS_RECORD*)(kaddr +
  5490. i * rec_size), rec_size);
  5491. kunmap_atomic(kaddr);
  5492. - local_irq_restore(flags);
  5493. + local_irq_restore_nort(flags);
  5494. flush_dcache_page(page);
  5495. if (likely(page_uptodate && !PageError(page)))
  5496. SetPageUptodate(page);
  5497. @@ -159,9 +157,7 @@
  5498. unlock_page(page);
  5499. return;
  5500. still_busy:
  5501. - bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
  5502. - local_irq_restore(flags);
  5503. - return;
  5504. + bh_uptodate_unlock_irqrestore(first, flags);
  5505. }
  5506. /**
  5507. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/proc/base.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/proc/base.c
  5508. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/proc/base.c 2017-04-16 10:38:22.000000000 +0200
  5509. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/proc/base.c 2017-04-18 17:54:25.000000000 +0200
  5510. @@ -1834,7 +1834,7 @@
  5511. child = d_hash_and_lookup(dir, &qname);
  5512. if (!child) {
  5513. - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
  5514. + DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(wq);
  5515. child = d_alloc_parallel(dir, &qname, &wq);
  5516. if (IS_ERR(child))
  5517. goto end_instantiate;
  5518. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/proc/proc_sysctl.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/proc/proc_sysctl.c
  5519. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/proc/proc_sysctl.c 2017-04-16 10:38:22.000000000 +0200
  5520. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/proc/proc_sysctl.c 2017-04-18 17:54:26.000000000 +0200
  5521. @@ -632,7 +632,7 @@
  5522. child = d_lookup(dir, &qname);
  5523. if (!child) {
  5524. - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
  5525. + DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(wq);
  5526. child = d_alloc_parallel(dir, &qname, &wq);
  5527. if (IS_ERR(child))
  5528. return false;
  5529. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/timerfd.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/timerfd.c
  5530. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/fs/timerfd.c 2017-04-16 10:38:22.000000000 +0200
  5531. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/fs/timerfd.c 2017-04-18 17:54:26.000000000 +0200
  5532. @@ -460,7 +460,10 @@
  5533. break;
  5534. }
  5535. spin_unlock_irq(&ctx->wqh.lock);
  5536. - cpu_relax();
  5537. + if (isalarm(ctx))
  5538. + hrtimer_wait_for_timer(&ctx->t.alarm.timer);
  5539. + else
  5540. + hrtimer_wait_for_timer(&ctx->t.tmr);
  5541. }
  5542. /*
  5543. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/acpi/platform/aclinux.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/acpi/platform/aclinux.h
  5544. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/acpi/platform/aclinux.h 2017-04-16 10:38:23.000000000 +0200
  5545. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/acpi/platform/aclinux.h 2017-04-18 17:54:26.000000000 +0200
  5546. @@ -133,6 +133,7 @@
  5547. #define acpi_cache_t struct kmem_cache
  5548. #define acpi_spinlock spinlock_t *
  5549. +#define acpi_raw_spinlock raw_spinlock_t *
  5550. #define acpi_cpu_flags unsigned long
  5551. /* Use native linux version of acpi_os_allocate_zeroed */
  5552. @@ -151,6 +152,20 @@
  5553. #define ACPI_USE_ALTERNATE_PROTOTYPE_acpi_os_get_thread_id
  5554. #define ACPI_USE_ALTERNATE_PROTOTYPE_acpi_os_create_lock
  5555. +#define acpi_os_create_raw_lock(__handle) \
  5556. +({ \
  5557. + raw_spinlock_t *lock = ACPI_ALLOCATE(sizeof(*lock)); \
  5558. + \
  5559. + if (lock) { \
  5560. + *(__handle) = lock; \
  5561. + raw_spin_lock_init(*(__handle)); \
  5562. + } \
  5563. + lock ? AE_OK : AE_NO_MEMORY; \
  5564. + })
  5565. +
  5566. +#define acpi_os_delete_raw_lock(__handle) kfree(__handle)
  5567. +
  5568. +
  5569. /*
  5570. * OSL interfaces used by debugger/disassembler
  5571. */
  5572. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/asm-generic/bug.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/asm-generic/bug.h
  5573. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/asm-generic/bug.h 2017-04-16 10:38:23.000000000 +0200
  5574. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/asm-generic/bug.h 2017-04-18 17:54:26.000000000 +0200
  5575. @@ -215,6 +215,20 @@
  5576. # define WARN_ON_SMP(x) ({0;})
  5577. #endif
  5578. +#ifdef CONFIG_PREEMPT_RT_BASE
  5579. +# define BUG_ON_RT(c) BUG_ON(c)
  5580. +# define BUG_ON_NONRT(c) do { } while (0)
  5581. +# define WARN_ON_RT(condition) WARN_ON(condition)
  5582. +# define WARN_ON_NONRT(condition) do { } while (0)
  5583. +# define WARN_ON_ONCE_NONRT(condition) do { } while (0)
  5584. +#else
  5585. +# define BUG_ON_RT(c) do { } while (0)
  5586. +# define BUG_ON_NONRT(c) BUG_ON(c)
  5587. +# define WARN_ON_RT(condition) do { } while (0)
  5588. +# define WARN_ON_NONRT(condition) WARN_ON(condition)
  5589. +# define WARN_ON_ONCE_NONRT(condition) WARN_ON_ONCE(condition)
  5590. +#endif
  5591. +
  5592. #endif /* __ASSEMBLY__ */
  5593. #endif
  5594. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/blk-mq.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/blk-mq.h
  5595. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/blk-mq.h 2017-04-16 10:38:24.000000000 +0200
  5596. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/blk-mq.h 2017-04-18 17:54:26.000000000 +0200
  5597. @@ -209,7 +209,7 @@
  5598. return unique_tag & BLK_MQ_UNIQUE_TAG_MASK;
  5599. }
  5600. -
  5601. +void __blk_mq_complete_request_remote_work(struct work_struct *work);
  5602. int blk_mq_request_started(struct request *rq);
  5603. void blk_mq_start_request(struct request *rq);
  5604. void blk_mq_end_request(struct request *rq, int error);
  5605. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/blkdev.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/blkdev.h
  5606. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/blkdev.h 2017-04-16 10:38:24.000000000 +0200
  5607. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/blkdev.h 2017-04-18 17:54:26.000000000 +0200
  5608. @@ -89,6 +89,7 @@
  5609. struct list_head queuelist;
  5610. union {
  5611. struct call_single_data csd;
  5612. + struct work_struct work;
  5613. u64 fifo_time;
  5614. };
  5615. @@ -467,7 +468,7 @@
  5616. struct throtl_data *td;
  5617. #endif
  5618. struct rcu_head rcu_head;
  5619. - wait_queue_head_t mq_freeze_wq;
  5620. + struct swait_queue_head mq_freeze_wq;
  5621. struct percpu_ref q_usage_counter;
  5622. struct list_head all_q_node;
  5623. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/bottom_half.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/bottom_half.h
  5624. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/bottom_half.h 2017-04-16 10:38:24.000000000 +0200
  5625. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/bottom_half.h 2017-04-18 17:54:26.000000000 +0200
  5626. @@ -3,6 +3,39 @@
  5627. #include <linux/preempt.h>
  5628. +#ifdef CONFIG_PREEMPT_RT_FULL
  5629. +
  5630. +extern void __local_bh_disable(void);
  5631. +extern void _local_bh_enable(void);
  5632. +extern void __local_bh_enable(void);
  5633. +
  5634. +static inline void local_bh_disable(void)
  5635. +{
  5636. + __local_bh_disable();
  5637. +}
  5638. +
  5639. +static inline void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
  5640. +{
  5641. + __local_bh_disable();
  5642. +}
  5643. +
  5644. +static inline void local_bh_enable(void)
  5645. +{
  5646. + __local_bh_enable();
  5647. +}
  5648. +
  5649. +static inline void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
  5650. +{
  5651. + __local_bh_enable();
  5652. +}
  5653. +
  5654. +static inline void local_bh_enable_ip(unsigned long ip)
  5655. +{
  5656. + __local_bh_enable();
  5657. +}
  5658. +
  5659. +#else
  5660. +
  5661. #ifdef CONFIG_TRACE_IRQFLAGS
  5662. extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt);
  5663. #else
  5664. @@ -30,5 +63,6 @@
  5665. {
  5666. __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET);
  5667. }
  5668. +#endif
  5669. #endif /* _LINUX_BH_H */
  5670. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/buffer_head.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/buffer_head.h
  5671. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/buffer_head.h 2017-04-16 10:38:24.000000000 +0200
  5672. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/buffer_head.h 2017-04-18 17:54:26.000000000 +0200
  5673. @@ -75,8 +75,50 @@
  5674. struct address_space *b_assoc_map; /* mapping this buffer is
  5675. associated with */
  5676. atomic_t b_count; /* users using this buffer_head */
  5677. +#ifdef CONFIG_PREEMPT_RT_BASE
  5678. + spinlock_t b_uptodate_lock;
  5679. +#if IS_ENABLED(CONFIG_JBD2)
  5680. + spinlock_t b_state_lock;
  5681. + spinlock_t b_journal_head_lock;
  5682. +#endif
  5683. +#endif
  5684. };
  5685. +static inline unsigned long bh_uptodate_lock_irqsave(struct buffer_head *bh)
  5686. +{
  5687. + unsigned long flags;
  5688. +
  5689. +#ifndef CONFIG_PREEMPT_RT_BASE
  5690. + local_irq_save(flags);
  5691. + bit_spin_lock(BH_Uptodate_Lock, &bh->b_state);
  5692. +#else
  5693. + spin_lock_irqsave(&bh->b_uptodate_lock, flags);
  5694. +#endif
  5695. + return flags;
  5696. +}
  5697. +
  5698. +static inline void
  5699. +bh_uptodate_unlock_irqrestore(struct buffer_head *bh, unsigned long flags)
  5700. +{
  5701. +#ifndef CONFIG_PREEMPT_RT_BASE
  5702. + bit_spin_unlock(BH_Uptodate_Lock, &bh->b_state);
  5703. + local_irq_restore(flags);
  5704. +#else
  5705. + spin_unlock_irqrestore(&bh->b_uptodate_lock, flags);
  5706. +#endif
  5707. +}
  5708. +
  5709. +static inline void buffer_head_init_locks(struct buffer_head *bh)
  5710. +{
  5711. +#ifdef CONFIG_PREEMPT_RT_BASE
  5712. + spin_lock_init(&bh->b_uptodate_lock);
  5713. +#if IS_ENABLED(CONFIG_JBD2)
  5714. + spin_lock_init(&bh->b_state_lock);
  5715. + spin_lock_init(&bh->b_journal_head_lock);
  5716. +#endif
  5717. +#endif
  5718. +}
  5719. +
  5720. /*
  5721. * macro tricks to expand the set_buffer_foo(), clear_buffer_foo()
  5722. * and buffer_foo() functions.
  5723. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/cgroup-defs.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/cgroup-defs.h
  5724. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/cgroup-defs.h 2017-04-16 10:38:24.000000000 +0200
  5725. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/cgroup-defs.h 2017-04-18 17:54:26.000000000 +0200
  5726. @@ -16,6 +16,7 @@
  5727. #include <linux/percpu-refcount.h>
  5728. #include <linux/percpu-rwsem.h>
  5729. #include <linux/workqueue.h>
  5730. +#include <linux/swork.h>
  5731. #ifdef CONFIG_CGROUPS
  5732. @@ -137,6 +138,7 @@
  5733. /* percpu_ref killing and RCU release */
  5734. struct rcu_head rcu_head;
  5735. struct work_struct destroy_work;
  5736. + struct swork_event destroy_swork;
  5737. };
  5738. /*
  5739. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/completion.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/completion.h
  5740. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/completion.h 2017-04-16 10:38:24.000000000 +0200
  5741. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/completion.h 2017-04-18 17:54:26.000000000 +0200
  5742. @@ -7,8 +7,7 @@
  5743. * Atomic wait-for-completion handler data structures.
  5744. * See kernel/sched/completion.c for details.
  5745. */
  5746. -
  5747. -#include <linux/wait.h>
  5748. +#include <linux/swait.h>
  5749. /*
  5750. * struct completion - structure used to maintain state for a "completion"
  5751. @@ -24,11 +23,11 @@
  5752. */
  5753. struct completion {
  5754. unsigned int done;
  5755. - wait_queue_head_t wait;
  5756. + struct swait_queue_head wait;
  5757. };
  5758. #define COMPLETION_INITIALIZER(work) \
  5759. - { 0, __WAIT_QUEUE_HEAD_INITIALIZER((work).wait) }
  5760. + { 0, __SWAIT_QUEUE_HEAD_INITIALIZER((work).wait) }
  5761. #define COMPLETION_INITIALIZER_ONSTACK(work) \
  5762. ({ init_completion(&work); work; })
  5763. @@ -73,7 +72,7 @@
  5764. static inline void init_completion(struct completion *x)
  5765. {
  5766. x->done = 0;
  5767. - init_waitqueue_head(&x->wait);
  5768. + init_swait_queue_head(&x->wait);
  5769. }
  5770. /**
  5771. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/cpu.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/cpu.h
  5772. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/cpu.h 2017-04-16 10:38:24.000000000 +0200
  5773. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/cpu.h 2017-04-18 17:54:26.000000000 +0200
  5774. @@ -182,6 +182,8 @@
  5775. extern void put_online_cpus(void);
  5776. extern void cpu_hotplug_disable(void);
  5777. extern void cpu_hotplug_enable(void);
  5778. +extern void pin_current_cpu(void);
  5779. +extern void unpin_current_cpu(void);
  5780. #define hotcpu_notifier(fn, pri) cpu_notifier(fn, pri)
  5781. #define __hotcpu_notifier(fn, pri) __cpu_notifier(fn, pri)
  5782. #define register_hotcpu_notifier(nb) register_cpu_notifier(nb)
  5783. @@ -199,6 +201,8 @@
  5784. #define put_online_cpus() do { } while (0)
  5785. #define cpu_hotplug_disable() do { } while (0)
  5786. #define cpu_hotplug_enable() do { } while (0)
  5787. +static inline void pin_current_cpu(void) { }
  5788. +static inline void unpin_current_cpu(void) { }
  5789. #define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0)
  5790. #define __hotcpu_notifier(fn, pri) do { (void)(fn); } while (0)
  5791. /* These aren't inline functions due to a GCC bug. */
  5792. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/dcache.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/dcache.h
  5793. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/dcache.h 2017-04-16 10:38:24.000000000 +0200
  5794. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/dcache.h 2017-04-18 17:54:26.000000000 +0200
  5795. @@ -11,6 +11,7 @@
  5796. #include <linux/rcupdate.h>
  5797. #include <linux/lockref.h>
  5798. #include <linux/stringhash.h>
  5799. +#include <linux/wait.h>
  5800. struct path;
  5801. struct vfsmount;
  5802. @@ -100,7 +101,7 @@
  5803. union {
  5804. struct list_head d_lru; /* LRU list */
  5805. - wait_queue_head_t *d_wait; /* in-lookup ones only */
  5806. + struct swait_queue_head *d_wait; /* in-lookup ones only */
  5807. };
  5808. struct list_head d_child; /* child of parent list */
  5809. struct list_head d_subdirs; /* our children */
  5810. @@ -230,7 +231,7 @@
  5811. extern struct dentry * d_alloc(struct dentry *, const struct qstr *);
  5812. extern struct dentry * d_alloc_pseudo(struct super_block *, const struct qstr *);
  5813. extern struct dentry * d_alloc_parallel(struct dentry *, const struct qstr *,
  5814. - wait_queue_head_t *);
  5815. + struct swait_queue_head *);
  5816. extern struct dentry * d_splice_alias(struct inode *, struct dentry *);
  5817. extern struct dentry * d_add_ci(struct dentry *, struct inode *, struct qstr *);
  5818. extern struct dentry * d_exact_alias(struct dentry *, struct inode *);
  5819. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/delay.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/delay.h
  5820. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/delay.h 2017-04-16 10:38:24.000000000 +0200
  5821. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/delay.h 2017-04-18 17:54:26.000000000 +0200
  5822. @@ -52,4 +52,10 @@
  5823. msleep(seconds * 1000);
  5824. }
  5825. +#ifdef CONFIG_PREEMPT_RT_FULL
  5826. +extern void cpu_chill(void);
  5827. +#else
  5828. +# define cpu_chill() cpu_relax()
  5829. +#endif
  5830. +
  5831. #endif /* defined(_LINUX_DELAY_H) */
  5832. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/highmem.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/highmem.h
  5833. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/highmem.h 2017-04-16 10:38:24.000000000 +0200
  5834. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/highmem.h 2017-04-18 17:54:26.000000000 +0200
  5835. @@ -7,6 +7,7 @@
  5836. #include <linux/mm.h>
  5837. #include <linux/uaccess.h>
  5838. #include <linux/hardirq.h>
  5839. +#include <linux/sched.h>
  5840. #include <asm/cacheflush.h>
  5841. @@ -65,7 +66,7 @@
  5842. static inline void *kmap_atomic(struct page *page)
  5843. {
  5844. - preempt_disable();
  5845. + preempt_disable_nort();
  5846. pagefault_disable();
  5847. return page_address(page);
  5848. }
  5849. @@ -74,7 +75,7 @@
  5850. static inline void __kunmap_atomic(void *addr)
  5851. {
  5852. pagefault_enable();
  5853. - preempt_enable();
  5854. + preempt_enable_nort();
  5855. }
  5856. #define kmap_atomic_pfn(pfn) kmap_atomic(pfn_to_page(pfn))
  5857. @@ -86,32 +87,51 @@
  5858. #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32)
  5859. +#ifndef CONFIG_PREEMPT_RT_FULL
  5860. DECLARE_PER_CPU(int, __kmap_atomic_idx);
  5861. +#endif
  5862. static inline int kmap_atomic_idx_push(void)
  5863. {
  5864. +#ifndef CONFIG_PREEMPT_RT_FULL
  5865. int idx = __this_cpu_inc_return(__kmap_atomic_idx) - 1;
  5866. -#ifdef CONFIG_DEBUG_HIGHMEM
  5867. +# ifdef CONFIG_DEBUG_HIGHMEM
  5868. WARN_ON_ONCE(in_irq() && !irqs_disabled());
  5869. BUG_ON(idx >= KM_TYPE_NR);
  5870. -#endif
  5871. +# endif
  5872. return idx;
  5873. +#else
  5874. + current->kmap_idx++;
  5875. + BUG_ON(current->kmap_idx > KM_TYPE_NR);
  5876. + return current->kmap_idx - 1;
  5877. +#endif
  5878. }
  5879. static inline int kmap_atomic_idx(void)
  5880. {
  5881. +#ifndef CONFIG_PREEMPT_RT_FULL
  5882. return __this_cpu_read(__kmap_atomic_idx) - 1;
  5883. +#else
  5884. + return current->kmap_idx - 1;
  5885. +#endif
  5886. }
  5887. static inline void kmap_atomic_idx_pop(void)
  5888. {
  5889. -#ifdef CONFIG_DEBUG_HIGHMEM
  5890. +#ifndef CONFIG_PREEMPT_RT_FULL
  5891. +# ifdef CONFIG_DEBUG_HIGHMEM
  5892. int idx = __this_cpu_dec_return(__kmap_atomic_idx);
  5893. BUG_ON(idx < 0);
  5894. -#else
  5895. +# else
  5896. __this_cpu_dec(__kmap_atomic_idx);
  5897. +# endif
  5898. +#else
  5899. + current->kmap_idx--;
  5900. +# ifdef CONFIG_DEBUG_HIGHMEM
  5901. + BUG_ON(current->kmap_idx < 0);
  5902. +# endif
  5903. #endif
  5904. }
  5905. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/hrtimer.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/hrtimer.h
  5906. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/hrtimer.h 2017-04-16 10:38:24.000000000 +0200
  5907. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/hrtimer.h 2017-04-18 17:54:26.000000000 +0200
  5908. @@ -87,6 +87,9 @@
  5909. * @function: timer expiry callback function
  5910. * @base: pointer to the timer base (per cpu and per clock)
  5911. * @state: state information (See bit values above)
  5912. + * @cb_entry: list entry to defer timers from hardirq context
  5913. + * @irqsafe: timer can run in hardirq context
  5914. + * @praecox: timer expiry time if expired at the time of programming
  5915. * @is_rel: Set if the timer was armed relative
  5916. * @start_pid: timer statistics field to store the pid of the task which
  5917. * started the timer
  5918. @@ -103,6 +106,11 @@
  5919. enum hrtimer_restart (*function)(struct hrtimer *);
  5920. struct hrtimer_clock_base *base;
  5921. u8 state;
  5922. + struct list_head cb_entry;
  5923. + int irqsafe;
  5924. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  5925. + ktime_t praecox;
  5926. +#endif
  5927. u8 is_rel;
  5928. #ifdef CONFIG_TIMER_STATS
  5929. int start_pid;
  5930. @@ -123,11 +131,7 @@
  5931. struct task_struct *task;
  5932. };
  5933. -#ifdef CONFIG_64BIT
  5934. # define HRTIMER_CLOCK_BASE_ALIGN 64
  5935. -#else
  5936. -# define HRTIMER_CLOCK_BASE_ALIGN 32
  5937. -#endif
  5938. /**
  5939. * struct hrtimer_clock_base - the timer base for a specific clock
  5940. @@ -136,6 +140,7 @@
  5941. * timer to a base on another cpu.
  5942. * @clockid: clock id for per_cpu support
  5943. * @active: red black tree root node for the active timers
  5944. + * @expired: list head for deferred timers.
  5945. * @get_time: function to retrieve the current time of the clock
  5946. * @offset: offset of this clock to the monotonic base
  5947. */
  5948. @@ -144,6 +149,7 @@
  5949. int index;
  5950. clockid_t clockid;
  5951. struct timerqueue_head active;
  5952. + struct list_head expired;
  5953. ktime_t (*get_time)(void);
  5954. ktime_t offset;
  5955. } __attribute__((__aligned__(HRTIMER_CLOCK_BASE_ALIGN)));
  5956. @@ -187,6 +193,7 @@
  5957. raw_spinlock_t lock;
  5958. seqcount_t seq;
  5959. struct hrtimer *running;
  5960. + struct hrtimer *running_soft;
  5961. unsigned int cpu;
  5962. unsigned int active_bases;
  5963. unsigned int clock_was_set_seq;
  5964. @@ -203,6 +210,9 @@
  5965. unsigned int nr_hangs;
  5966. unsigned int max_hang_time;
  5967. #endif
  5968. +#ifdef CONFIG_PREEMPT_RT_BASE
  5969. + wait_queue_head_t wait;
  5970. +#endif
  5971. struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES];
  5972. } ____cacheline_aligned;
  5973. @@ -412,6 +422,13 @@
  5974. hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
  5975. }
  5976. +/* Softirq preemption could deadlock timer removal */
  5977. +#ifdef CONFIG_PREEMPT_RT_BASE
  5978. + extern void hrtimer_wait_for_timer(const struct hrtimer *timer);
  5979. +#else
  5980. +# define hrtimer_wait_for_timer(timer) do { cpu_relax(); } while (0)
  5981. +#endif
  5982. +
  5983. /* Query timers: */
  5984. extern ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust);
  5985. @@ -436,9 +453,15 @@
  5986. * Helper function to check, whether the timer is running the callback
  5987. * function
  5988. */
  5989. -static inline int hrtimer_callback_running(struct hrtimer *timer)
  5990. +static inline int hrtimer_callback_running(const struct hrtimer *timer)
  5991. {
  5992. - return timer->base->cpu_base->running == timer;
  5993. + if (timer->base->cpu_base->running == timer)
  5994. + return 1;
  5995. +#ifdef CONFIG_PREEMPT_RT_BASE
  5996. + if (timer->base->cpu_base->running_soft == timer)
  5997. + return 1;
  5998. +#endif
  5999. + return 0;
  6000. }
  6001. /* Forward a hrtimer so it expires after now: */
  6002. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/idr.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/idr.h
  6003. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/idr.h 2017-04-16 10:38:24.000000000 +0200
  6004. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/idr.h 2017-04-18 17:54:26.000000000 +0200
  6005. @@ -95,10 +95,14 @@
  6006. * Each idr_preload() should be matched with an invocation of this
  6007. * function. See idr_preload() for details.
  6008. */
  6009. +#ifdef CONFIG_PREEMPT_RT_FULL
  6010. +void idr_preload_end(void);
  6011. +#else
  6012. static inline void idr_preload_end(void)
  6013. {
  6014. preempt_enable();
  6015. }
  6016. +#endif
  6017. /**
  6018. * idr_find - return pointer for given id
  6019. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/init_task.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/init_task.h
  6020. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/init_task.h 2017-04-16 10:38:24.000000000 +0200
  6021. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/init_task.h 2017-04-18 17:54:26.000000000 +0200
  6022. @@ -150,6 +150,12 @@
  6023. # define INIT_PERF_EVENTS(tsk)
  6024. #endif
  6025. +#ifdef CONFIG_PREEMPT_RT_BASE
  6026. +# define INIT_TIMER_LIST .posix_timer_list = NULL,
  6027. +#else
  6028. +# define INIT_TIMER_LIST
  6029. +#endif
  6030. +
  6031. #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
  6032. # define INIT_VTIME(tsk) \
  6033. .vtime_seqcount = SEQCNT_ZERO(tsk.vtime_seqcount), \
  6034. @@ -250,6 +256,7 @@
  6035. .cpu_timers = INIT_CPU_TIMERS(tsk.cpu_timers), \
  6036. .pi_lock = __RAW_SPIN_LOCK_UNLOCKED(tsk.pi_lock), \
  6037. .timer_slack_ns = 50000, /* 50 usec default slack */ \
  6038. + INIT_TIMER_LIST \
  6039. .pids = { \
  6040. [PIDTYPE_PID] = INIT_PID_LINK(PIDTYPE_PID), \
  6041. [PIDTYPE_PGID] = INIT_PID_LINK(PIDTYPE_PGID), \
  6042. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/interrupt.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/interrupt.h
  6043. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/interrupt.h 2017-04-16 10:38:24.000000000 +0200
  6044. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/interrupt.h 2017-04-18 17:54:26.000000000 +0200
  6045. @@ -14,6 +14,7 @@
  6046. #include <linux/hrtimer.h>
  6047. #include <linux/kref.h>
  6048. #include <linux/workqueue.h>
  6049. +#include <linux/swork.h>
  6050. #include <linux/atomic.h>
  6051. #include <asm/ptrace.h>
  6052. @@ -61,6 +62,7 @@
  6053. * interrupt handler after suspending interrupts. For system
  6054. * wakeup devices users need to implement wakeup detection in
  6055. * their interrupt handlers.
  6056. + * IRQF_NO_SOFTIRQ_CALL - Do not process softirqs in the irq thread context (RT)
  6057. */
  6058. #define IRQF_SHARED 0x00000080
  6059. #define IRQF_PROBE_SHARED 0x00000100
  6060. @@ -74,6 +76,7 @@
  6061. #define IRQF_NO_THREAD 0x00010000
  6062. #define IRQF_EARLY_RESUME 0x00020000
  6063. #define IRQF_COND_SUSPEND 0x00040000
  6064. +#define IRQF_NO_SOFTIRQ_CALL 0x00080000
  6065. #define IRQF_TIMER (__IRQF_TIMER | IRQF_NO_SUSPEND | IRQF_NO_THREAD)
  6066. @@ -196,7 +199,7 @@
  6067. #ifdef CONFIG_LOCKDEP
  6068. # define local_irq_enable_in_hardirq() do { } while (0)
  6069. #else
  6070. -# define local_irq_enable_in_hardirq() local_irq_enable()
  6071. +# define local_irq_enable_in_hardirq() local_irq_enable_nort()
  6072. #endif
  6073. extern void disable_irq_nosync(unsigned int irq);
  6074. @@ -216,6 +219,7 @@
  6075. * struct irq_affinity_notify - context for notification of IRQ affinity changes
  6076. * @irq: Interrupt to which notification applies
  6077. * @kref: Reference count, for internal use
  6078. + * @swork: Swork item, for internal use
  6079. * @work: Work item, for internal use
  6080. * @notify: Function to be called on change. This will be
  6081. * called in process context.
  6082. @@ -227,7 +231,11 @@
  6083. struct irq_affinity_notify {
  6084. unsigned int irq;
  6085. struct kref kref;
  6086. +#ifdef CONFIG_PREEMPT_RT_BASE
  6087. + struct swork_event swork;
  6088. +#else
  6089. struct work_struct work;
  6090. +#endif
  6091. void (*notify)(struct irq_affinity_notify *, const cpumask_t *mask);
  6092. void (*release)(struct kref *ref);
  6093. };
  6094. @@ -406,9 +414,13 @@
  6095. bool state);
  6096. #ifdef CONFIG_IRQ_FORCED_THREADING
  6097. +# ifndef CONFIG_PREEMPT_RT_BASE
  6098. extern bool force_irqthreads;
  6099. +# else
  6100. +# define force_irqthreads (true)
  6101. +# endif
  6102. #else
  6103. -#define force_irqthreads (0)
  6104. +#define force_irqthreads (false)
  6105. #endif
  6106. #ifndef __ARCH_SET_SOFTIRQ_PENDING
  6107. @@ -465,9 +477,10 @@
  6108. void (*action)(struct softirq_action *);
  6109. };
  6110. +#ifndef CONFIG_PREEMPT_RT_FULL
  6111. asmlinkage void do_softirq(void);
  6112. asmlinkage void __do_softirq(void);
  6113. -
  6114. +static inline void thread_do_softirq(void) { do_softirq(); }
  6115. #ifdef __ARCH_HAS_DO_SOFTIRQ
  6116. void do_softirq_own_stack(void);
  6117. #else
  6118. @@ -476,13 +489,25 @@
  6119. __do_softirq();
  6120. }
  6121. #endif
  6122. +#else
  6123. +extern void thread_do_softirq(void);
  6124. +#endif
  6125. extern void open_softirq(int nr, void (*action)(struct softirq_action *));
  6126. extern void softirq_init(void);
  6127. extern void __raise_softirq_irqoff(unsigned int nr);
  6128. +#ifdef CONFIG_PREEMPT_RT_FULL
  6129. +extern void __raise_softirq_irqoff_ksoft(unsigned int nr);
  6130. +#else
  6131. +static inline void __raise_softirq_irqoff_ksoft(unsigned int nr)
  6132. +{
  6133. + __raise_softirq_irqoff(nr);
  6134. +}
  6135. +#endif
  6136. extern void raise_softirq_irqoff(unsigned int nr);
  6137. extern void raise_softirq(unsigned int nr);
  6138. +extern void softirq_check_pending_idle(void);
  6139. DECLARE_PER_CPU(struct task_struct *, ksoftirqd);
  6140. @@ -504,8 +529,9 @@
  6141. to be executed on some cpu at least once after this.
  6142. * If the tasklet is already scheduled, but its execution is still not
  6143. started, it will be executed only once.
  6144. - * If this tasklet is already running on another CPU (or schedule is called
  6145. - from tasklet itself), it is rescheduled for later.
  6146. + * If this tasklet is already running on another CPU, it is rescheduled
  6147. + for later.
  6148. + * Schedule must not be called from the tasklet itself (a lockup occurs)
  6149. * Tasklet is strictly serialized wrt itself, but not
  6150. wrt another tasklets. If client needs some intertask synchronization,
  6151. he makes it with spinlocks.
  6152. @@ -530,27 +556,36 @@
  6153. enum
  6154. {
  6155. TASKLET_STATE_SCHED, /* Tasklet is scheduled for execution */
  6156. - TASKLET_STATE_RUN /* Tasklet is running (SMP only) */
  6157. + TASKLET_STATE_RUN, /* Tasklet is running (SMP only) */
  6158. + TASKLET_STATE_PENDING /* Tasklet is pending */
  6159. };
  6160. -#ifdef CONFIG_SMP
  6161. +#define TASKLET_STATEF_SCHED (1 << TASKLET_STATE_SCHED)
  6162. +#define TASKLET_STATEF_RUN (1 << TASKLET_STATE_RUN)
  6163. +#define TASKLET_STATEF_PENDING (1 << TASKLET_STATE_PENDING)
  6164. +
  6165. +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
  6166. static inline int tasklet_trylock(struct tasklet_struct *t)
  6167. {
  6168. return !test_and_set_bit(TASKLET_STATE_RUN, &(t)->state);
  6169. }
  6170. +static inline int tasklet_tryunlock(struct tasklet_struct *t)
  6171. +{
  6172. + return cmpxchg(&t->state, TASKLET_STATEF_RUN, 0) == TASKLET_STATEF_RUN;
  6173. +}
  6174. +
  6175. static inline void tasklet_unlock(struct tasklet_struct *t)
  6176. {
  6177. smp_mb__before_atomic();
  6178. clear_bit(TASKLET_STATE_RUN, &(t)->state);
  6179. }
  6180. -static inline void tasklet_unlock_wait(struct tasklet_struct *t)
  6181. -{
  6182. - while (test_bit(TASKLET_STATE_RUN, &(t)->state)) { barrier(); }
  6183. -}
  6184. +extern void tasklet_unlock_wait(struct tasklet_struct *t);
  6185. +
  6186. #else
  6187. #define tasklet_trylock(t) 1
  6188. +#define tasklet_tryunlock(t) 1
  6189. #define tasklet_unlock_wait(t) do { } while (0)
  6190. #define tasklet_unlock(t) do { } while (0)
  6191. #endif
  6192. @@ -599,12 +634,7 @@
  6193. smp_mb();
  6194. }
  6195. -static inline void tasklet_enable(struct tasklet_struct *t)
  6196. -{
  6197. - smp_mb__before_atomic();
  6198. - atomic_dec(&t->count);
  6199. -}
  6200. -
  6201. +extern void tasklet_enable(struct tasklet_struct *t);
  6202. extern void tasklet_kill(struct tasklet_struct *t);
  6203. extern void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu);
  6204. extern void tasklet_init(struct tasklet_struct *t,
  6205. @@ -635,6 +665,12 @@
  6206. tasklet_kill(&ttimer->tasklet);
  6207. }
  6208. +#ifdef CONFIG_PREEMPT_RT_FULL
  6209. +extern void softirq_early_init(void);
  6210. +#else
  6211. +static inline void softirq_early_init(void) { }
  6212. +#endif
  6213. +
  6214. /*
  6215. * Autoprobing for irqs:
  6216. *
  6217. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/irq.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/irq.h
  6218. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/irq.h 2017-04-16 10:38:24.000000000 +0200
  6219. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/irq.h 2017-04-18 17:54:26.000000000 +0200
  6220. @@ -72,6 +72,7 @@
  6221. * IRQ_IS_POLLED - Always polled by another interrupt. Exclude
  6222. * it from the spurious interrupt detection
  6223. * mechanism and from core side polling.
  6224. + * IRQ_NO_SOFTIRQ_CALL - No softirq processing in the irq thread context (RT)
  6225. * IRQ_DISABLE_UNLAZY - Disable lazy irq disable
  6226. */
  6227. enum {
  6228. @@ -99,13 +100,14 @@
  6229. IRQ_PER_CPU_DEVID = (1 << 17),
  6230. IRQ_IS_POLLED = (1 << 18),
  6231. IRQ_DISABLE_UNLAZY = (1 << 19),
  6232. + IRQ_NO_SOFTIRQ_CALL = (1 << 20),
  6233. };
  6234. #define IRQF_MODIFY_MASK \
  6235. (IRQ_TYPE_SENSE_MASK | IRQ_NOPROBE | IRQ_NOREQUEST | \
  6236. IRQ_NOAUTOEN | IRQ_MOVE_PCNTXT | IRQ_LEVEL | IRQ_NO_BALANCING | \
  6237. IRQ_PER_CPU | IRQ_NESTED_THREAD | IRQ_NOTHREAD | IRQ_PER_CPU_DEVID | \
  6238. - IRQ_IS_POLLED | IRQ_DISABLE_UNLAZY)
  6239. + IRQ_IS_POLLED | IRQ_DISABLE_UNLAZY | IRQ_NO_SOFTIRQ_CALL)
  6240. #define IRQ_NO_BALANCING_MASK (IRQ_PER_CPU | IRQ_NO_BALANCING)
  6241. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/irq_work.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/irq_work.h
  6242. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/irq_work.h 2017-04-16 10:38:24.000000000 +0200
  6243. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/irq_work.h 2017-04-18 17:54:26.000000000 +0200
  6244. @@ -16,6 +16,7 @@
  6245. #define IRQ_WORK_BUSY 2UL
  6246. #define IRQ_WORK_FLAGS 3UL
  6247. #define IRQ_WORK_LAZY 4UL /* Doesn't want IPI, wait for tick */
  6248. +#define IRQ_WORK_HARD_IRQ 8UL /* Run hard IRQ context, even on RT */
  6249. struct irq_work {
  6250. unsigned long flags;
  6251. @@ -51,4 +52,10 @@
  6252. static inline void irq_work_run(void) { }
  6253. #endif
  6254. +#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_PREEMPT_RT_FULL)
  6255. +void irq_work_tick_soft(void);
  6256. +#else
  6257. +static inline void irq_work_tick_soft(void) { }
  6258. +#endif
  6259. +
  6260. #endif /* _LINUX_IRQ_WORK_H */
  6261. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/irqdesc.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/irqdesc.h
  6262. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/irqdesc.h 2017-04-16 10:38:24.000000000 +0200
  6263. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/irqdesc.h 2017-04-18 17:54:26.000000000 +0200
  6264. @@ -66,6 +66,7 @@
  6265. unsigned int irqs_unhandled;
  6266. atomic_t threads_handled;
  6267. int threads_handled_last;
  6268. + u64 random_ip;
  6269. raw_spinlock_t lock;
  6270. struct cpumask *percpu_enabled;
  6271. const struct cpumask *percpu_affinity;
  6272. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/irqflags.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/irqflags.h
  6273. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/irqflags.h 2017-04-16 10:38:24.000000000 +0200
  6274. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/irqflags.h 2017-04-18 17:54:26.000000000 +0200
  6275. @@ -25,8 +25,6 @@
  6276. # define trace_softirqs_enabled(p) ((p)->softirqs_enabled)
  6277. # define trace_hardirq_enter() do { current->hardirq_context++; } while (0)
  6278. # define trace_hardirq_exit() do { current->hardirq_context--; } while (0)
  6279. -# define lockdep_softirq_enter() do { current->softirq_context++; } while (0)
  6280. -# define lockdep_softirq_exit() do { current->softirq_context--; } while (0)
  6281. # define INIT_TRACE_IRQFLAGS .softirqs_enabled = 1,
  6282. #else
  6283. # define trace_hardirqs_on() do { } while (0)
  6284. @@ -39,9 +37,15 @@
  6285. # define trace_softirqs_enabled(p) 0
  6286. # define trace_hardirq_enter() do { } while (0)
  6287. # define trace_hardirq_exit() do { } while (0)
  6288. +# define INIT_TRACE_IRQFLAGS
  6289. +#endif
  6290. +
  6291. +#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_PREEMPT_RT_FULL)
  6292. +# define lockdep_softirq_enter() do { current->softirq_context++; } while (0)
  6293. +# define lockdep_softirq_exit() do { current->softirq_context--; } while (0)
  6294. +#else
  6295. # define lockdep_softirq_enter() do { } while (0)
  6296. # define lockdep_softirq_exit() do { } while (0)
  6297. -# define INIT_TRACE_IRQFLAGS
  6298. #endif
  6299. #if defined(CONFIG_IRQSOFF_TRACER) || \
  6300. @@ -148,4 +152,23 @@
  6301. #define irqs_disabled_flags(flags) raw_irqs_disabled_flags(flags)
  6302. +/*
  6303. + * local_irq* variants depending on RT/!RT
  6304. + */
  6305. +#ifdef CONFIG_PREEMPT_RT_FULL
  6306. +# define local_irq_disable_nort() do { } while (0)
  6307. +# define local_irq_enable_nort() do { } while (0)
  6308. +# define local_irq_save_nort(flags) local_save_flags(flags)
  6309. +# define local_irq_restore_nort(flags) (void)(flags)
  6310. +# define local_irq_disable_rt() local_irq_disable()
  6311. +# define local_irq_enable_rt() local_irq_enable()
  6312. +#else
  6313. +# define local_irq_disable_nort() local_irq_disable()
  6314. +# define local_irq_enable_nort() local_irq_enable()
  6315. +# define local_irq_save_nort(flags) local_irq_save(flags)
  6316. +# define local_irq_restore_nort(flags) local_irq_restore(flags)
  6317. +# define local_irq_disable_rt() do { } while (0)
  6318. +# define local_irq_enable_rt() do { } while (0)
  6319. +#endif
  6320. +
  6321. #endif
  6322. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/jbd2.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/jbd2.h
  6323. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/jbd2.h 2017-04-16 10:38:24.000000000 +0200
  6324. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/jbd2.h 2017-04-18 17:54:26.000000000 +0200
  6325. @@ -347,32 +347,56 @@
  6326. static inline void jbd_lock_bh_state(struct buffer_head *bh)
  6327. {
  6328. +#ifndef CONFIG_PREEMPT_RT_BASE
  6329. bit_spin_lock(BH_State, &bh->b_state);
  6330. +#else
  6331. + spin_lock(&bh->b_state_lock);
  6332. +#endif
  6333. }
  6334. static inline int jbd_trylock_bh_state(struct buffer_head *bh)
  6335. {
  6336. +#ifndef CONFIG_PREEMPT_RT_BASE
  6337. return bit_spin_trylock(BH_State, &bh->b_state);
  6338. +#else
  6339. + return spin_trylock(&bh->b_state_lock);
  6340. +#endif
  6341. }
  6342. static inline int jbd_is_locked_bh_state(struct buffer_head *bh)
  6343. {
  6344. +#ifndef CONFIG_PREEMPT_RT_BASE
  6345. return bit_spin_is_locked(BH_State, &bh->b_state);
  6346. +#else
  6347. + return spin_is_locked(&bh->b_state_lock);
  6348. +#endif
  6349. }
  6350. static inline void jbd_unlock_bh_state(struct buffer_head *bh)
  6351. {
  6352. +#ifndef CONFIG_PREEMPT_RT_BASE
  6353. bit_spin_unlock(BH_State, &bh->b_state);
  6354. +#else
  6355. + spin_unlock(&bh->b_state_lock);
  6356. +#endif
  6357. }
  6358. static inline void jbd_lock_bh_journal_head(struct buffer_head *bh)
  6359. {
  6360. +#ifndef CONFIG_PREEMPT_RT_BASE
  6361. bit_spin_lock(BH_JournalHead, &bh->b_state);
  6362. +#else
  6363. + spin_lock(&bh->b_journal_head_lock);
  6364. +#endif
  6365. }
  6366. static inline void jbd_unlock_bh_journal_head(struct buffer_head *bh)
  6367. {
  6368. +#ifndef CONFIG_PREEMPT_RT_BASE
  6369. bit_spin_unlock(BH_JournalHead, &bh->b_state);
  6370. +#else
  6371. + spin_unlock(&bh->b_journal_head_lock);
  6372. +#endif
  6373. }
  6374. #define J_ASSERT(assert) BUG_ON(!(assert))
  6375. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/kdb.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/kdb.h
  6376. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/kdb.h 2017-04-16 10:38:24.000000000 +0200
  6377. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/kdb.h 2017-04-18 17:54:26.000000000 +0200
  6378. @@ -167,6 +167,7 @@
  6379. extern __printf(1, 2) int kdb_printf(const char *, ...);
  6380. typedef __printf(1, 2) int (*kdb_printf_t)(const char *, ...);
  6381. +#define in_kdb_printk() (kdb_trap_printk)
  6382. extern void kdb_init(int level);
  6383. /* Access to kdb specific polling devices */
  6384. @@ -201,6 +202,7 @@
  6385. extern int kdb_unregister(char *);
  6386. #else /* ! CONFIG_KGDB_KDB */
  6387. static inline __printf(1, 2) int kdb_printf(const char *fmt, ...) { return 0; }
  6388. +#define in_kdb_printk() (0)
  6389. static inline void kdb_init(int level) {}
  6390. static inline int kdb_register(char *cmd, kdb_func_t func, char *usage,
  6391. char *help, short minlen) { return 0; }
  6392. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/kernel.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/kernel.h
  6393. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/kernel.h 2017-04-16 10:38:24.000000000 +0200
  6394. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/kernel.h 2017-04-18 17:54:26.000000000 +0200
  6395. @@ -194,6 +194,9 @@
  6396. */
  6397. # define might_sleep() \
  6398. do { __might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0)
  6399. +
  6400. +# define might_sleep_no_state_check() \
  6401. + do { ___might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0)
  6402. # define sched_annotate_sleep() (current->task_state_change = 0)
  6403. #else
  6404. static inline void ___might_sleep(const char *file, int line,
  6405. @@ -201,6 +204,7 @@
  6406. static inline void __might_sleep(const char *file, int line,
  6407. int preempt_offset) { }
  6408. # define might_sleep() do { might_resched(); } while (0)
  6409. +# define might_sleep_no_state_check() do { might_resched(); } while (0)
  6410. # define sched_annotate_sleep() do { } while (0)
  6411. #endif
  6412. @@ -488,6 +492,7 @@
  6413. SYSTEM_HALT,
  6414. SYSTEM_POWER_OFF,
  6415. SYSTEM_RESTART,
  6416. + SYSTEM_SUSPEND,
  6417. } system_state;
  6418. #define TAINT_PROPRIETARY_MODULE 0
  6419. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/list_bl.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/list_bl.h
  6420. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/list_bl.h 2017-04-16 10:38:25.000000000 +0200
  6421. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/list_bl.h 2017-04-18 17:54:26.000000000 +0200
  6422. @@ -2,6 +2,7 @@
  6423. #define _LINUX_LIST_BL_H
  6424. #include <linux/list.h>
  6425. +#include <linux/spinlock.h>
  6426. #include <linux/bit_spinlock.h>
  6427. /*
  6428. @@ -32,13 +33,24 @@
  6429. struct hlist_bl_head {
  6430. struct hlist_bl_node *first;
  6431. +#ifdef CONFIG_PREEMPT_RT_BASE
  6432. + raw_spinlock_t lock;
  6433. +#endif
  6434. };
  6435. struct hlist_bl_node {
  6436. struct hlist_bl_node *next, **pprev;
  6437. };
  6438. -#define INIT_HLIST_BL_HEAD(ptr) \
  6439. - ((ptr)->first = NULL)
  6440. +
  6441. +#ifdef CONFIG_PREEMPT_RT_BASE
  6442. +#define INIT_HLIST_BL_HEAD(h) \
  6443. +do { \
  6444. + (h)->first = NULL; \
  6445. + raw_spin_lock_init(&(h)->lock); \
  6446. +} while (0)
  6447. +#else
  6448. +#define INIT_HLIST_BL_HEAD(h) (h)->first = NULL
  6449. +#endif
  6450. static inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h)
  6451. {
  6452. @@ -118,12 +130,26 @@
  6453. static inline void hlist_bl_lock(struct hlist_bl_head *b)
  6454. {
  6455. +#ifndef CONFIG_PREEMPT_RT_BASE
  6456. bit_spin_lock(0, (unsigned long *)b);
  6457. +#else
  6458. + raw_spin_lock(&b->lock);
  6459. +#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
  6460. + __set_bit(0, (unsigned long *)b);
  6461. +#endif
  6462. +#endif
  6463. }
  6464. static inline void hlist_bl_unlock(struct hlist_bl_head *b)
  6465. {
  6466. +#ifndef CONFIG_PREEMPT_RT_BASE
  6467. __bit_spin_unlock(0, (unsigned long *)b);
  6468. +#else
  6469. +#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
  6470. + __clear_bit(0, (unsigned long *)b);
  6471. +#endif
  6472. + raw_spin_unlock(&b->lock);
  6473. +#endif
  6474. }
  6475. static inline bool hlist_bl_is_locked(struct hlist_bl_head *b)
  6476. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/locallock.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/locallock.h
  6477. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/locallock.h 1970-01-01 01:00:00.000000000 +0100
  6478. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/locallock.h 2017-04-18 17:54:26.000000000 +0200
  6479. @@ -0,0 +1,278 @@
  6480. +#ifndef _LINUX_LOCALLOCK_H
  6481. +#define _LINUX_LOCALLOCK_H
  6482. +
  6483. +#include <linux/percpu.h>
  6484. +#include <linux/spinlock.h>
  6485. +
  6486. +#ifdef CONFIG_PREEMPT_RT_BASE
  6487. +
  6488. +#ifdef CONFIG_DEBUG_SPINLOCK
  6489. +# define LL_WARN(cond) WARN_ON(cond)
  6490. +#else
  6491. +# define LL_WARN(cond) do { } while (0)
  6492. +#endif
  6493. +
  6494. +/*
  6495. + * per cpu lock based substitute for local_irq_*()
  6496. + */
  6497. +struct local_irq_lock {
  6498. + spinlock_t lock;
  6499. + struct task_struct *owner;
  6500. + int nestcnt;
  6501. + unsigned long flags;
  6502. +};
  6503. +
  6504. +#define DEFINE_LOCAL_IRQ_LOCK(lvar) \
  6505. + DEFINE_PER_CPU(struct local_irq_lock, lvar) = { \
  6506. + .lock = __SPIN_LOCK_UNLOCKED((lvar).lock) }
  6507. +
  6508. +#define DECLARE_LOCAL_IRQ_LOCK(lvar) \
  6509. + DECLARE_PER_CPU(struct local_irq_lock, lvar)
  6510. +
  6511. +#define local_irq_lock_init(lvar) \
  6512. + do { \
  6513. + int __cpu; \
  6514. + for_each_possible_cpu(__cpu) \
  6515. + spin_lock_init(&per_cpu(lvar, __cpu).lock); \
  6516. + } while (0)
  6517. +
  6518. +/*
  6519. + * spin_lock|trylock|unlock_local flavour that does not migrate disable
  6520. + * used for __local_lock|trylock|unlock where get_local_var/put_local_var
  6521. + * already takes care of the migrate_disable/enable
  6522. + * for CONFIG_PREEMPT_BASE map to the normal spin_* calls.
  6523. + */
  6524. +#ifdef CONFIG_PREEMPT_RT_FULL
  6525. +# define spin_lock_local(lock) rt_spin_lock__no_mg(lock)
  6526. +# define spin_trylock_local(lock) rt_spin_trylock__no_mg(lock)
  6527. +# define spin_unlock_local(lock) rt_spin_unlock__no_mg(lock)
  6528. +#else
  6529. +# define spin_lock_local(lock) spin_lock(lock)
  6530. +# define spin_trylock_local(lock) spin_trylock(lock)
  6531. +# define spin_unlock_local(lock) spin_unlock(lock)
  6532. +#endif
  6533. +
  6534. +static inline void __local_lock(struct local_irq_lock *lv)
  6535. +{
  6536. + if (lv->owner != current) {
  6537. + spin_lock_local(&lv->lock);
  6538. + LL_WARN(lv->owner);
  6539. + LL_WARN(lv->nestcnt);
  6540. + lv->owner = current;
  6541. + }
  6542. + lv->nestcnt++;
  6543. +}
  6544. +
  6545. +#define local_lock(lvar) \
  6546. + do { __local_lock(&get_local_var(lvar)); } while (0)
  6547. +
  6548. +#define local_lock_on(lvar, cpu) \
  6549. + do { __local_lock(&per_cpu(lvar, cpu)); } while (0)
  6550. +
  6551. +static inline int __local_trylock(struct local_irq_lock *lv)
  6552. +{
  6553. + if (lv->owner != current && spin_trylock_local(&lv->lock)) {
  6554. + LL_WARN(lv->owner);
  6555. + LL_WARN(lv->nestcnt);
  6556. + lv->owner = current;
  6557. + lv->nestcnt = 1;
  6558. + return 1;
  6559. + }
  6560. + return 0;
  6561. +}
  6562. +
  6563. +#define local_trylock(lvar) \
  6564. + ({ \
  6565. + int __locked; \
  6566. + __locked = __local_trylock(&get_local_var(lvar)); \
  6567. + if (!__locked) \
  6568. + put_local_var(lvar); \
  6569. + __locked; \
  6570. + })
  6571. +
  6572. +static inline void __local_unlock(struct local_irq_lock *lv)
  6573. +{
  6574. + LL_WARN(lv->nestcnt == 0);
  6575. + LL_WARN(lv->owner != current);
  6576. + if (--lv->nestcnt)
  6577. + return;
  6578. +
  6579. + lv->owner = NULL;
  6580. + spin_unlock_local(&lv->lock);
  6581. +}
  6582. +
  6583. +#define local_unlock(lvar) \
  6584. + do { \
  6585. + __local_unlock(this_cpu_ptr(&lvar)); \
  6586. + put_local_var(lvar); \
  6587. + } while (0)
  6588. +
  6589. +#define local_unlock_on(lvar, cpu) \
  6590. + do { __local_unlock(&per_cpu(lvar, cpu)); } while (0)
  6591. +
  6592. +static inline void __local_lock_irq(struct local_irq_lock *lv)
  6593. +{
  6594. + spin_lock_irqsave(&lv->lock, lv->flags);
  6595. + LL_WARN(lv->owner);
  6596. + LL_WARN(lv->nestcnt);
  6597. + lv->owner = current;
  6598. + lv->nestcnt = 1;
  6599. +}
  6600. +
  6601. +#define local_lock_irq(lvar) \
  6602. + do { __local_lock_irq(&get_local_var(lvar)); } while (0)
  6603. +
  6604. +#define local_lock_irq_on(lvar, cpu) \
  6605. + do { __local_lock_irq(&per_cpu(lvar, cpu)); } while (0)
  6606. +
  6607. +static inline void __local_unlock_irq(struct local_irq_lock *lv)
  6608. +{
  6609. + LL_WARN(!lv->nestcnt);
  6610. + LL_WARN(lv->owner != current);
  6611. + lv->owner = NULL;
  6612. + lv->nestcnt = 0;
  6613. + spin_unlock_irq(&lv->lock);
  6614. +}
  6615. +
  6616. +#define local_unlock_irq(lvar) \
  6617. + do { \
  6618. + __local_unlock_irq(this_cpu_ptr(&lvar)); \
  6619. + put_local_var(lvar); \
  6620. + } while (0)
  6621. +
  6622. +#define local_unlock_irq_on(lvar, cpu) \
  6623. + do { \
  6624. + __local_unlock_irq(&per_cpu(lvar, cpu)); \
  6625. + } while (0)
  6626. +
  6627. +static inline int __local_lock_irqsave(struct local_irq_lock *lv)
  6628. +{
  6629. + if (lv->owner != current) {
  6630. + __local_lock_irq(lv);
  6631. + return 0;
  6632. + } else {
  6633. + lv->nestcnt++;
  6634. + return 1;
  6635. + }
  6636. +}
  6637. +
  6638. +#define local_lock_irqsave(lvar, _flags) \
  6639. + do { \
  6640. + if (__local_lock_irqsave(&get_local_var(lvar))) \
  6641. + put_local_var(lvar); \
  6642. + _flags = __this_cpu_read(lvar.flags); \
  6643. + } while (0)
  6644. +
  6645. +#define local_lock_irqsave_on(lvar, _flags, cpu) \
  6646. + do { \
  6647. + __local_lock_irqsave(&per_cpu(lvar, cpu)); \
  6648. + _flags = per_cpu(lvar, cpu).flags; \
  6649. + } while (0)
  6650. +
  6651. +static inline int __local_unlock_irqrestore(struct local_irq_lock *lv,
  6652. + unsigned long flags)
  6653. +{
  6654. + LL_WARN(!lv->nestcnt);
  6655. + LL_WARN(lv->owner != current);
  6656. + if (--lv->nestcnt)
  6657. + return 0;
  6658. +
  6659. + lv->owner = NULL;
  6660. + spin_unlock_irqrestore(&lv->lock, lv->flags);
  6661. + return 1;
  6662. +}
  6663. +
  6664. +#define local_unlock_irqrestore(lvar, flags) \
  6665. + do { \
  6666. + if (__local_unlock_irqrestore(this_cpu_ptr(&lvar), flags)) \
  6667. + put_local_var(lvar); \
  6668. + } while (0)
  6669. +
  6670. +#define local_unlock_irqrestore_on(lvar, flags, cpu) \
  6671. + do { \
  6672. + __local_unlock_irqrestore(&per_cpu(lvar, cpu), flags); \
  6673. + } while (0)
  6674. +
  6675. +#define local_spin_trylock_irq(lvar, lock) \
  6676. + ({ \
  6677. + int __locked; \
  6678. + local_lock_irq(lvar); \
  6679. + __locked = spin_trylock(lock); \
  6680. + if (!__locked) \
  6681. + local_unlock_irq(lvar); \
  6682. + __locked; \
  6683. + })
  6684. +
  6685. +#define local_spin_lock_irq(lvar, lock) \
  6686. + do { \
  6687. + local_lock_irq(lvar); \
  6688. + spin_lock(lock); \
  6689. + } while (0)
  6690. +
  6691. +#define local_spin_unlock_irq(lvar, lock) \
  6692. + do { \
  6693. + spin_unlock(lock); \
  6694. + local_unlock_irq(lvar); \
  6695. + } while (0)
  6696. +
  6697. +#define local_spin_lock_irqsave(lvar, lock, flags) \
  6698. + do { \
  6699. + local_lock_irqsave(lvar, flags); \
  6700. + spin_lock(lock); \
  6701. + } while (0)
  6702. +
  6703. +#define local_spin_unlock_irqrestore(lvar, lock, flags) \
  6704. + do { \
  6705. + spin_unlock(lock); \
  6706. + local_unlock_irqrestore(lvar, flags); \
  6707. + } while (0)
  6708. +
  6709. +#define get_locked_var(lvar, var) \
  6710. + (*({ \
  6711. + local_lock(lvar); \
  6712. + this_cpu_ptr(&var); \
  6713. + }))
  6714. +
  6715. +#define put_locked_var(lvar, var) local_unlock(lvar);
  6716. +
  6717. +#define local_lock_cpu(lvar) \
  6718. + ({ \
  6719. + local_lock(lvar); \
  6720. + smp_processor_id(); \
  6721. + })
  6722. +
  6723. +#define local_unlock_cpu(lvar) local_unlock(lvar)
  6724. +
  6725. +#else /* PREEMPT_RT_BASE */
  6726. +
  6727. +#define DEFINE_LOCAL_IRQ_LOCK(lvar) __typeof__(const int) lvar
  6728. +#define DECLARE_LOCAL_IRQ_LOCK(lvar) extern __typeof__(const int) lvar
  6729. +
  6730. +static inline void local_irq_lock_init(int lvar) { }
  6731. +
  6732. +#define local_lock(lvar) preempt_disable()
  6733. +#define local_unlock(lvar) preempt_enable()
  6734. +#define local_lock_irq(lvar) local_irq_disable()
  6735. +#define local_lock_irq_on(lvar, cpu) local_irq_disable()
  6736. +#define local_unlock_irq(lvar) local_irq_enable()
  6737. +#define local_unlock_irq_on(lvar, cpu) local_irq_enable()
  6738. +#define local_lock_irqsave(lvar, flags) local_irq_save(flags)
  6739. +#define local_unlock_irqrestore(lvar, flags) local_irq_restore(flags)
  6740. +
  6741. +#define local_spin_trylock_irq(lvar, lock) spin_trylock_irq(lock)
  6742. +#define local_spin_lock_irq(lvar, lock) spin_lock_irq(lock)
  6743. +#define local_spin_unlock_irq(lvar, lock) spin_unlock_irq(lock)
  6744. +#define local_spin_lock_irqsave(lvar, lock, flags) \
  6745. + spin_lock_irqsave(lock, flags)
  6746. +#define local_spin_unlock_irqrestore(lvar, lock, flags) \
  6747. + spin_unlock_irqrestore(lock, flags)
  6748. +
  6749. +#define get_locked_var(lvar, var) get_cpu_var(var)
  6750. +#define put_locked_var(lvar, var) put_cpu_var(var)
  6751. +
  6752. +#define local_lock_cpu(lvar) get_cpu()
  6753. +#define local_unlock_cpu(lvar) put_cpu()
  6754. +
  6755. +#endif
  6756. +
  6757. +#endif
  6758. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/mm_types.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/mm_types.h
  6759. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/mm_types.h 2017-04-16 10:38:25.000000000 +0200
  6760. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/mm_types.h 2017-04-18 17:54:26.000000000 +0200
  6761. @@ -11,6 +11,7 @@
  6762. #include <linux/completion.h>
  6763. #include <linux/cpumask.h>
  6764. #include <linux/uprobes.h>
  6765. +#include <linux/rcupdate.h>
  6766. #include <linux/page-flags-layout.h>
  6767. #include <linux/workqueue.h>
  6768. #include <asm/page.h>
  6769. @@ -509,6 +510,9 @@
  6770. bool tlb_flush_pending;
  6771. #endif
  6772. struct uprobes_state uprobes_state;
  6773. +#ifdef CONFIG_PREEMPT_RT_BASE
  6774. + struct rcu_head delayed_drop;
  6775. +#endif
  6776. #ifdef CONFIG_X86_INTEL_MPX
  6777. /* address of the bounds directory */
  6778. void __user *bd_addr;
  6779. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/module.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/module.h
  6780. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/module.h 2017-04-16 10:38:25.000000000 +0200
  6781. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/module.h 2017-04-18 17:54:26.000000000 +0200
  6782. @@ -496,6 +496,7 @@
  6783. struct module *__module_text_address(unsigned long addr);
  6784. struct module *__module_address(unsigned long addr);
  6785. bool is_module_address(unsigned long addr);
  6786. +bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr);
  6787. bool is_module_percpu_address(unsigned long addr);
  6788. bool is_module_text_address(unsigned long addr);
  6789. @@ -663,6 +664,11 @@
  6790. return false;
  6791. }
  6792. +static inline bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
  6793. +{
  6794. + return false;
  6795. +}
  6796. +
  6797. static inline bool is_module_text_address(unsigned long addr)
  6798. {
  6799. return false;
  6800. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/mutex.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/mutex.h
  6801. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/mutex.h 2017-04-16 10:38:25.000000000 +0200
  6802. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/mutex.h 2017-04-18 17:54:26.000000000 +0200
  6803. @@ -19,6 +19,17 @@
  6804. #include <asm/processor.h>
  6805. #include <linux/osq_lock.h>
  6806. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  6807. +# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \
  6808. + , .dep_map = { .name = #lockname }
  6809. +#else
  6810. +# define __DEP_MAP_MUTEX_INITIALIZER(lockname)
  6811. +#endif
  6812. +
  6813. +#ifdef CONFIG_PREEMPT_RT_FULL
  6814. +# include <linux/mutex_rt.h>
  6815. +#else
  6816. +
  6817. /*
  6818. * Simple, straightforward mutexes with strict semantics:
  6819. *
  6820. @@ -99,13 +110,6 @@
  6821. static inline void mutex_destroy(struct mutex *lock) {}
  6822. #endif
  6823. -#ifdef CONFIG_DEBUG_LOCK_ALLOC
  6824. -# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \
  6825. - , .dep_map = { .name = #lockname }
  6826. -#else
  6827. -# define __DEP_MAP_MUTEX_INITIALIZER(lockname)
  6828. -#endif
  6829. -
  6830. #define __MUTEX_INITIALIZER(lockname) \
  6831. { .count = ATOMIC_INIT(1) \
  6832. , .wait_lock = __SPIN_LOCK_UNLOCKED(lockname.wait_lock) \
  6833. @@ -173,6 +177,8 @@
  6834. extern int mutex_trylock(struct mutex *lock);
  6835. extern void mutex_unlock(struct mutex *lock);
  6836. +#endif /* !PREEMPT_RT_FULL */
  6837. +
  6838. extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock);
  6839. #endif /* __LINUX_MUTEX_H */
  6840. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/mutex_rt.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/mutex_rt.h
  6841. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/mutex_rt.h 1970-01-01 01:00:00.000000000 +0100
  6842. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/mutex_rt.h 2017-04-18 17:54:26.000000000 +0200
  6843. @@ -0,0 +1,89 @@
  6844. +#ifndef __LINUX_MUTEX_RT_H
  6845. +#define __LINUX_MUTEX_RT_H
  6846. +
  6847. +#ifndef __LINUX_MUTEX_H
  6848. +#error "Please include mutex.h"
  6849. +#endif
  6850. +
  6851. +#include <linux/rtmutex.h>
  6852. +
  6853. +/* FIXME: Just for __lockfunc */
  6854. +#include <linux/spinlock.h>
  6855. +
  6856. +struct mutex {
  6857. + struct rt_mutex lock;
  6858. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  6859. + struct lockdep_map dep_map;
  6860. +#endif
  6861. +};
  6862. +
  6863. +#define __MUTEX_INITIALIZER(mutexname) \
  6864. + { \
  6865. + .lock = __RT_MUTEX_INITIALIZER(mutexname.lock) \
  6866. + __DEP_MAP_MUTEX_INITIALIZER(mutexname) \
  6867. + }
  6868. +
  6869. +#define DEFINE_MUTEX(mutexname) \
  6870. + struct mutex mutexname = __MUTEX_INITIALIZER(mutexname)
  6871. +
  6872. +extern void __mutex_do_init(struct mutex *lock, const char *name, struct lock_class_key *key);
  6873. +extern void __lockfunc _mutex_lock(struct mutex *lock);
  6874. +extern int __lockfunc _mutex_lock_interruptible(struct mutex *lock);
  6875. +extern int __lockfunc _mutex_lock_killable(struct mutex *lock);
  6876. +extern void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass);
  6877. +extern void __lockfunc _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock);
  6878. +extern int __lockfunc _mutex_lock_interruptible_nested(struct mutex *lock, int subclass);
  6879. +extern int __lockfunc _mutex_lock_killable_nested(struct mutex *lock, int subclass);
  6880. +extern int __lockfunc _mutex_trylock(struct mutex *lock);
  6881. +extern void __lockfunc _mutex_unlock(struct mutex *lock);
  6882. +
  6883. +#define mutex_is_locked(l) rt_mutex_is_locked(&(l)->lock)
  6884. +#define mutex_lock(l) _mutex_lock(l)
  6885. +#define mutex_lock_interruptible(l) _mutex_lock_interruptible(l)
  6886. +#define mutex_lock_killable(l) _mutex_lock_killable(l)
  6887. +#define mutex_trylock(l) _mutex_trylock(l)
  6888. +#define mutex_unlock(l) _mutex_unlock(l)
  6889. +
  6890. +#ifdef CONFIG_DEBUG_MUTEXES
  6891. +#define mutex_destroy(l) rt_mutex_destroy(&(l)->lock)
  6892. +#else
  6893. +static inline void mutex_destroy(struct mutex *lock) {}
  6894. +#endif
  6895. +
  6896. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  6897. +# define mutex_lock_nested(l, s) _mutex_lock_nested(l, s)
  6898. +# define mutex_lock_interruptible_nested(l, s) \
  6899. + _mutex_lock_interruptible_nested(l, s)
  6900. +# define mutex_lock_killable_nested(l, s) \
  6901. + _mutex_lock_killable_nested(l, s)
  6902. +
  6903. +# define mutex_lock_nest_lock(lock, nest_lock) \
  6904. +do { \
  6905. + typecheck(struct lockdep_map *, &(nest_lock)->dep_map); \
  6906. + _mutex_lock_nest_lock(lock, &(nest_lock)->dep_map); \
  6907. +} while (0)
  6908. +
  6909. +#else
  6910. +# define mutex_lock_nested(l, s) _mutex_lock(l)
  6911. +# define mutex_lock_interruptible_nested(l, s) \
  6912. + _mutex_lock_interruptible(l)
  6913. +# define mutex_lock_killable_nested(l, s) \
  6914. + _mutex_lock_killable(l)
  6915. +# define mutex_lock_nest_lock(lock, nest_lock) mutex_lock(lock)
  6916. +#endif
  6917. +
  6918. +# define mutex_init(mutex) \
  6919. +do { \
  6920. + static struct lock_class_key __key; \
  6921. + \
  6922. + rt_mutex_init(&(mutex)->lock); \
  6923. + __mutex_do_init((mutex), #mutex, &__key); \
  6924. +} while (0)
  6925. +
  6926. +# define __mutex_init(mutex, name, key) \
  6927. +do { \
  6928. + rt_mutex_init(&(mutex)->lock); \
  6929. + __mutex_do_init((mutex), name, key); \
  6930. +} while (0)
  6931. +
  6932. +#endif
  6933. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/netdevice.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/netdevice.h
  6934. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/netdevice.h 2017-04-16 10:38:25.000000000 +0200
  6935. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/netdevice.h 2017-04-18 17:54:26.000000000 +0200
  6936. @@ -396,7 +396,19 @@
  6937. typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
  6938. void __napi_schedule(struct napi_struct *n);
  6939. +
  6940. +/*
  6941. + * When PREEMPT_RT_FULL is defined, all device interrupt handlers
  6942. + * run as threads, and they can also be preempted (without PREEMPT_RT
  6943. + * interrupt threads can not be preempted). Which means that calling
  6944. + * __napi_schedule_irqoff() from an interrupt handler can be preempted
  6945. + * and can corrupt the napi->poll_list.
  6946. + */
  6947. +#ifdef CONFIG_PREEMPT_RT_FULL
  6948. +#define __napi_schedule_irqoff(n) __napi_schedule(n)
  6949. +#else
  6950. void __napi_schedule_irqoff(struct napi_struct *n);
  6951. +#endif
  6952. static inline bool napi_disable_pending(struct napi_struct *n)
  6953. {
  6954. @@ -2463,14 +2475,53 @@
  6955. void synchronize_net(void);
  6956. int init_dummy_netdev(struct net_device *dev);
  6957. -DECLARE_PER_CPU(int, xmit_recursion);
  6958. #define XMIT_RECURSION_LIMIT 10
  6959. +#ifdef CONFIG_PREEMPT_RT_FULL
  6960. +static inline int dev_recursion_level(void)
  6961. +{
  6962. + return current->xmit_recursion;
  6963. +}
  6964. +
  6965. +static inline int xmit_rec_read(void)
  6966. +{
  6967. + return current->xmit_recursion;
  6968. +}
  6969. +
  6970. +static inline void xmit_rec_inc(void)
  6971. +{
  6972. + current->xmit_recursion++;
  6973. +}
  6974. +
  6975. +static inline void xmit_rec_dec(void)
  6976. +{
  6977. + current->xmit_recursion--;
  6978. +}
  6979. +
  6980. +#else
  6981. +
  6982. +DECLARE_PER_CPU(int, xmit_recursion);
  6983. static inline int dev_recursion_level(void)
  6984. {
  6985. return this_cpu_read(xmit_recursion);
  6986. }
  6987. +static inline int xmit_rec_read(void)
  6988. +{
  6989. + return __this_cpu_read(xmit_recursion);
  6990. +}
  6991. +
  6992. +static inline void xmit_rec_inc(void)
  6993. +{
  6994. + __this_cpu_inc(xmit_recursion);
  6995. +}
  6996. +
  6997. +static inline void xmit_rec_dec(void)
  6998. +{
  6999. + __this_cpu_dec(xmit_recursion);
  7000. +}
  7001. +#endif
  7002. +
  7003. struct net_device *dev_get_by_index(struct net *net, int ifindex);
  7004. struct net_device *__dev_get_by_index(struct net *net, int ifindex);
  7005. struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
  7006. @@ -2855,6 +2906,7 @@
  7007. unsigned int dropped;
  7008. struct sk_buff_head input_pkt_queue;
  7009. struct napi_struct backlog;
  7010. + struct sk_buff_head tofree_queue;
  7011. };
  7012. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/netfilter/x_tables.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/netfilter/x_tables.h
  7013. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/netfilter/x_tables.h 2017-04-16 10:38:25.000000000 +0200
  7014. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/netfilter/x_tables.h 2017-04-18 17:54:26.000000000 +0200
  7015. @@ -4,6 +4,7 @@
  7016. #include <linux/netdevice.h>
  7017. #include <linux/static_key.h>
  7018. +#include <linux/locallock.h>
  7019. #include <uapi/linux/netfilter/x_tables.h>
  7020. /* Test a struct->invflags and a boolean for inequality */
  7021. @@ -300,6 +301,8 @@
  7022. */
  7023. DECLARE_PER_CPU(seqcount_t, xt_recseq);
  7024. +DECLARE_LOCAL_IRQ_LOCK(xt_write_lock);
  7025. +
  7026. /* xt_tee_enabled - true if x_tables needs to handle reentrancy
  7027. *
  7028. * Enabled if current ip(6)tables ruleset has at least one -j TEE rule.
  7029. @@ -320,6 +323,9 @@
  7030. {
  7031. unsigned int addend;
  7032. + /* RT protection */
  7033. + local_lock(xt_write_lock);
  7034. +
  7035. /*
  7036. * Low order bit of sequence is set if we already
  7037. * called xt_write_recseq_begin().
  7038. @@ -350,6 +356,7 @@
  7039. /* this is kind of a write_seqcount_end(), but addend is 0 or 1 */
  7040. smp_wmb();
  7041. __this_cpu_add(xt_recseq.sequence, addend);
  7042. + local_unlock(xt_write_lock);
  7043. }
  7044. /*
  7045. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/nfs_fs.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/nfs_fs.h
  7046. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/nfs_fs.h 2017-04-16 10:38:25.000000000 +0200
  7047. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/nfs_fs.h 2017-04-18 17:54:26.000000000 +0200
  7048. @@ -165,7 +165,11 @@
  7049. /* Readers: in-flight sillydelete RPC calls */
  7050. /* Writers: rmdir */
  7051. +#ifdef CONFIG_PREEMPT_RT_BASE
  7052. + struct semaphore rmdir_sem;
  7053. +#else
  7054. struct rw_semaphore rmdir_sem;
  7055. +#endif
  7056. #if IS_ENABLED(CONFIG_NFS_V4)
  7057. struct nfs4_cached_acl *nfs4_acl;
  7058. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/nfs_xdr.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/nfs_xdr.h
  7059. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/nfs_xdr.h 2017-04-16 10:38:25.000000000 +0200
  7060. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/nfs_xdr.h 2017-04-18 17:54:26.000000000 +0200
  7061. @@ -1490,7 +1490,7 @@
  7062. struct nfs_removeargs args;
  7063. struct nfs_removeres res;
  7064. struct dentry *dentry;
  7065. - wait_queue_head_t wq;
  7066. + struct swait_queue_head wq;
  7067. struct rpc_cred *cred;
  7068. struct nfs_fattr dir_attr;
  7069. long timeout;
  7070. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/notifier.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/notifier.h
  7071. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/notifier.h 2017-04-16 10:38:25.000000000 +0200
  7072. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/notifier.h 2017-04-18 17:54:26.000000000 +0200
  7073. @@ -6,7 +6,7 @@
  7074. *
  7075. * Alan Cox <Alan.Cox@linux.org>
  7076. */
  7077. -
  7078. +
  7079. #ifndef _LINUX_NOTIFIER_H
  7080. #define _LINUX_NOTIFIER_H
  7081. #include <linux/errno.h>
  7082. @@ -42,9 +42,7 @@
  7083. * in srcu_notifier_call_chain(): no cache bounces and no memory barriers.
  7084. * As compensation, srcu_notifier_chain_unregister() is rather expensive.
  7085. * SRCU notifier chains should be used when the chain will be called very
  7086. - * often but notifier_blocks will seldom be removed. Also, SRCU notifier
  7087. - * chains are slightly more difficult to use because they require special
  7088. - * runtime initialization.
  7089. + * often but notifier_blocks will seldom be removed.
  7090. */
  7091. struct notifier_block;
  7092. @@ -90,7 +88,7 @@
  7093. (name)->head = NULL; \
  7094. } while (0)
  7095. -/* srcu_notifier_heads must be initialized and cleaned up dynamically */
  7096. +/* srcu_notifier_heads must be cleaned up dynamically */
  7097. extern void srcu_init_notifier_head(struct srcu_notifier_head *nh);
  7098. #define srcu_cleanup_notifier_head(name) \
  7099. cleanup_srcu_struct(&(name)->srcu);
  7100. @@ -103,7 +101,13 @@
  7101. .head = NULL }
  7102. #define RAW_NOTIFIER_INIT(name) { \
  7103. .head = NULL }
  7104. -/* srcu_notifier_heads cannot be initialized statically */
  7105. +
  7106. +#define SRCU_NOTIFIER_INIT(name, pcpu) \
  7107. + { \
  7108. + .mutex = __MUTEX_INITIALIZER(name.mutex), \
  7109. + .head = NULL, \
  7110. + .srcu = __SRCU_STRUCT_INIT(name.srcu, pcpu), \
  7111. + }
  7112. #define ATOMIC_NOTIFIER_HEAD(name) \
  7113. struct atomic_notifier_head name = \
  7114. @@ -115,6 +119,18 @@
  7115. struct raw_notifier_head name = \
  7116. RAW_NOTIFIER_INIT(name)
  7117. +#define _SRCU_NOTIFIER_HEAD(name, mod) \
  7118. + static DEFINE_PER_CPU(struct srcu_struct_array, \
  7119. + name##_head_srcu_array); \
  7120. + mod struct srcu_notifier_head name = \
  7121. + SRCU_NOTIFIER_INIT(name, name##_head_srcu_array)
  7122. +
  7123. +#define SRCU_NOTIFIER_HEAD(name) \
  7124. + _SRCU_NOTIFIER_HEAD(name, )
  7125. +
  7126. +#define SRCU_NOTIFIER_HEAD_STATIC(name) \
  7127. + _SRCU_NOTIFIER_HEAD(name, static)
  7128. +
  7129. #ifdef __KERNEL__
  7130. extern int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
  7131. @@ -184,12 +200,12 @@
  7132. /*
  7133. * Declared notifiers so far. I can imagine quite a few more chains
  7134. - * over time (eg laptop power reset chains, reboot chain (to clean
  7135. + * over time (eg laptop power reset chains, reboot chain (to clean
  7136. * device units up), device [un]mount chain, module load/unload chain,
  7137. - * low memory chain, screenblank chain (for plug in modular screenblankers)
  7138. + * low memory chain, screenblank chain (for plug in modular screenblankers)
  7139. * VC switch chains (for loadable kernel svgalib VC switch helpers) etc...
  7140. */
  7141. -
  7142. +
  7143. /* CPU notfiers are defined in include/linux/cpu.h. */
  7144. /* netdevice notifiers are defined in include/linux/netdevice.h */
  7145. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/percpu-rwsem.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/percpu-rwsem.h
  7146. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/percpu-rwsem.h 2017-04-16 10:38:25.000000000 +0200
  7147. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/percpu-rwsem.h 2017-04-18 17:54:26.000000000 +0200
  7148. @@ -4,7 +4,7 @@
  7149. #include <linux/atomic.h>
  7150. #include <linux/rwsem.h>
  7151. #include <linux/percpu.h>
  7152. -#include <linux/wait.h>
  7153. +#include <linux/swait.h>
  7154. #include <linux/rcu_sync.h>
  7155. #include <linux/lockdep.h>
  7156. @@ -12,7 +12,7 @@
  7157. struct rcu_sync rss;
  7158. unsigned int __percpu *read_count;
  7159. struct rw_semaphore rw_sem;
  7160. - wait_queue_head_t writer;
  7161. + struct swait_queue_head writer;
  7162. int readers_block;
  7163. };
  7164. @@ -22,13 +22,13 @@
  7165. .rss = __RCU_SYNC_INITIALIZER(name.rss, RCU_SCHED_SYNC), \
  7166. .read_count = &__percpu_rwsem_rc_##name, \
  7167. .rw_sem = __RWSEM_INITIALIZER(name.rw_sem), \
  7168. - .writer = __WAIT_QUEUE_HEAD_INITIALIZER(name.writer), \
  7169. + .writer = __SWAIT_QUEUE_HEAD_INITIALIZER(name.writer), \
  7170. }
  7171. extern int __percpu_down_read(struct percpu_rw_semaphore *, int);
  7172. extern void __percpu_up_read(struct percpu_rw_semaphore *);
  7173. -static inline void percpu_down_read_preempt_disable(struct percpu_rw_semaphore *sem)
  7174. +static inline void percpu_down_read(struct percpu_rw_semaphore *sem)
  7175. {
  7176. might_sleep();
  7177. @@ -46,16 +46,10 @@
  7178. __this_cpu_inc(*sem->read_count);
  7179. if (unlikely(!rcu_sync_is_idle(&sem->rss)))
  7180. __percpu_down_read(sem, false); /* Unconditional memory barrier */
  7181. - barrier();
  7182. /*
  7183. - * The barrier() prevents the compiler from
  7184. + * The preempt_enable() prevents the compiler from
  7185. * bleeding the critical section out.
  7186. */
  7187. -}
  7188. -
  7189. -static inline void percpu_down_read(struct percpu_rw_semaphore *sem)
  7190. -{
  7191. - percpu_down_read_preempt_disable(sem);
  7192. preempt_enable();
  7193. }
  7194. @@ -82,13 +76,9 @@
  7195. return ret;
  7196. }
  7197. -static inline void percpu_up_read_preempt_enable(struct percpu_rw_semaphore *sem)
  7198. +static inline void percpu_up_read(struct percpu_rw_semaphore *sem)
  7199. {
  7200. - /*
  7201. - * The barrier() prevents the compiler from
  7202. - * bleeding the critical section out.
  7203. - */
  7204. - barrier();
  7205. + preempt_disable();
  7206. /*
  7207. * Same as in percpu_down_read().
  7208. */
  7209. @@ -101,12 +91,6 @@
  7210. rwsem_release(&sem->rw_sem.dep_map, 1, _RET_IP_);
  7211. }
  7212. -static inline void percpu_up_read(struct percpu_rw_semaphore *sem)
  7213. -{
  7214. - preempt_disable();
  7215. - percpu_up_read_preempt_enable(sem);
  7216. -}
  7217. -
  7218. extern void percpu_down_write(struct percpu_rw_semaphore *);
  7219. extern void percpu_up_write(struct percpu_rw_semaphore *);
  7220. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/percpu.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/percpu.h
  7221. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/percpu.h 2017-04-16 10:38:25.000000000 +0200
  7222. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/percpu.h 2017-04-18 17:54:26.000000000 +0200
  7223. @@ -18,6 +18,35 @@
  7224. #define PERCPU_MODULE_RESERVE 0
  7225. #endif
  7226. +#ifdef CONFIG_PREEMPT_RT_FULL
  7227. +
  7228. +#define get_local_var(var) (*({ \
  7229. + migrate_disable(); \
  7230. + this_cpu_ptr(&var); }))
  7231. +
  7232. +#define put_local_var(var) do { \
  7233. + (void)&(var); \
  7234. + migrate_enable(); \
  7235. +} while (0)
  7236. +
  7237. +# define get_local_ptr(var) ({ \
  7238. + migrate_disable(); \
  7239. + this_cpu_ptr(var); })
  7240. +
  7241. +# define put_local_ptr(var) do { \
  7242. + (void)(var); \
  7243. + migrate_enable(); \
  7244. +} while (0)
  7245. +
  7246. +#else
  7247. +
  7248. +#define get_local_var(var) get_cpu_var(var)
  7249. +#define put_local_var(var) put_cpu_var(var)
  7250. +#define get_local_ptr(var) get_cpu_ptr(var)
  7251. +#define put_local_ptr(var) put_cpu_ptr(var)
  7252. +
  7253. +#endif
  7254. +
  7255. /* minimum unit size, also is the maximum supported allocation size */
  7256. #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10)
  7257. @@ -110,6 +139,7 @@
  7258. #endif
  7259. extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
  7260. +extern bool __is_kernel_percpu_address(unsigned long addr, unsigned long *can_addr);
  7261. extern bool is_kernel_percpu_address(unsigned long addr);
  7262. #if !defined(CONFIG_SMP) || !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA)
  7263. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/pid.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/pid.h
  7264. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/pid.h 2017-04-16 10:38:25.000000000 +0200
  7265. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/pid.h 2017-04-18 17:54:26.000000000 +0200
  7266. @@ -2,6 +2,7 @@
  7267. #define _LINUX_PID_H
  7268. #include <linux/rcupdate.h>
  7269. +#include <linux/atomic.h>
  7270. enum pid_type
  7271. {
  7272. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/preempt.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/preempt.h
  7273. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/preempt.h 2017-04-16 10:38:26.000000000 +0200
  7274. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/preempt.h 2017-04-18 17:54:26.000000000 +0200
  7275. @@ -50,7 +50,11 @@
  7276. #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT)
  7277. #define NMI_OFFSET (1UL << NMI_SHIFT)
  7278. -#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET)
  7279. +#ifndef CONFIG_PREEMPT_RT_FULL
  7280. +# define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET)
  7281. +#else
  7282. +# define SOFTIRQ_DISABLE_OFFSET (0)
  7283. +#endif
  7284. /* We use the MSB mostly because its available */
  7285. #define PREEMPT_NEED_RESCHED 0x80000000
  7286. @@ -59,9 +63,15 @@
  7287. #include <asm/preempt.h>
  7288. #define hardirq_count() (preempt_count() & HARDIRQ_MASK)
  7289. -#define softirq_count() (preempt_count() & SOFTIRQ_MASK)
  7290. #define irq_count() (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \
  7291. | NMI_MASK))
  7292. +#ifndef CONFIG_PREEMPT_RT_FULL
  7293. +# define softirq_count() (preempt_count() & SOFTIRQ_MASK)
  7294. +# define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET)
  7295. +#else
  7296. +# define softirq_count() (0UL)
  7297. +extern int in_serving_softirq(void);
  7298. +#endif
  7299. /*
  7300. * Are we doing bottom half or hardware interrupt processing?
  7301. @@ -72,7 +82,6 @@
  7302. #define in_irq() (hardirq_count())
  7303. #define in_softirq() (softirq_count())
  7304. #define in_interrupt() (irq_count())
  7305. -#define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET)
  7306. /*
  7307. * Are we in NMI context?
  7308. @@ -91,7 +100,11 @@
  7309. /*
  7310. * The preempt_count offset after spin_lock()
  7311. */
  7312. +#if !defined(CONFIG_PREEMPT_RT_FULL)
  7313. #define PREEMPT_LOCK_OFFSET PREEMPT_DISABLE_OFFSET
  7314. +#else
  7315. +#define PREEMPT_LOCK_OFFSET 0
  7316. +#endif
  7317. /*
  7318. * The preempt_count offset needed for things like:
  7319. @@ -140,6 +153,20 @@
  7320. #define preempt_count_inc() preempt_count_add(1)
  7321. #define preempt_count_dec() preempt_count_sub(1)
  7322. +#ifdef CONFIG_PREEMPT_LAZY
  7323. +#define add_preempt_lazy_count(val) do { preempt_lazy_count() += (val); } while (0)
  7324. +#define sub_preempt_lazy_count(val) do { preempt_lazy_count() -= (val); } while (0)
  7325. +#define inc_preempt_lazy_count() add_preempt_lazy_count(1)
  7326. +#define dec_preempt_lazy_count() sub_preempt_lazy_count(1)
  7327. +#define preempt_lazy_count() (current_thread_info()->preempt_lazy_count)
  7328. +#else
  7329. +#define add_preempt_lazy_count(val) do { } while (0)
  7330. +#define sub_preempt_lazy_count(val) do { } while (0)
  7331. +#define inc_preempt_lazy_count() do { } while (0)
  7332. +#define dec_preempt_lazy_count() do { } while (0)
  7333. +#define preempt_lazy_count() (0)
  7334. +#endif
  7335. +
  7336. #ifdef CONFIG_PREEMPT_COUNT
  7337. #define preempt_disable() \
  7338. @@ -148,13 +175,25 @@
  7339. barrier(); \
  7340. } while (0)
  7341. +#define preempt_lazy_disable() \
  7342. +do { \
  7343. + inc_preempt_lazy_count(); \
  7344. + barrier(); \
  7345. +} while (0)
  7346. +
  7347. #define sched_preempt_enable_no_resched() \
  7348. do { \
  7349. barrier(); \
  7350. preempt_count_dec(); \
  7351. } while (0)
  7352. -#define preempt_enable_no_resched() sched_preempt_enable_no_resched()
  7353. +#ifdef CONFIG_PREEMPT_RT_BASE
  7354. +# define preempt_enable_no_resched() sched_preempt_enable_no_resched()
  7355. +# define preempt_check_resched_rt() preempt_check_resched()
  7356. +#else
  7357. +# define preempt_enable_no_resched() preempt_enable()
  7358. +# define preempt_check_resched_rt() barrier();
  7359. +#endif
  7360. #define preemptible() (preempt_count() == 0 && !irqs_disabled())
  7361. @@ -179,6 +218,13 @@
  7362. __preempt_schedule(); \
  7363. } while (0)
  7364. +#define preempt_lazy_enable() \
  7365. +do { \
  7366. + dec_preempt_lazy_count(); \
  7367. + barrier(); \
  7368. + preempt_check_resched(); \
  7369. +} while (0)
  7370. +
  7371. #else /* !CONFIG_PREEMPT */
  7372. #define preempt_enable() \
  7373. do { \
  7374. @@ -224,6 +270,7 @@
  7375. #define preempt_disable_notrace() barrier()
  7376. #define preempt_enable_no_resched_notrace() barrier()
  7377. #define preempt_enable_notrace() barrier()
  7378. +#define preempt_check_resched_rt() barrier()
  7379. #define preemptible() 0
  7380. #endif /* CONFIG_PREEMPT_COUNT */
  7381. @@ -244,10 +291,31 @@
  7382. } while (0)
  7383. #define preempt_fold_need_resched() \
  7384. do { \
  7385. - if (tif_need_resched()) \
  7386. + if (tif_need_resched_now()) \
  7387. set_preempt_need_resched(); \
  7388. } while (0)
  7389. +#ifdef CONFIG_PREEMPT_RT_FULL
  7390. +# define preempt_disable_rt() preempt_disable()
  7391. +# define preempt_enable_rt() preempt_enable()
  7392. +# define preempt_disable_nort() barrier()
  7393. +# define preempt_enable_nort() barrier()
  7394. +# ifdef CONFIG_SMP
  7395. + extern void migrate_disable(void);
  7396. + extern void migrate_enable(void);
  7397. +# else /* CONFIG_SMP */
  7398. +# define migrate_disable() barrier()
  7399. +# define migrate_enable() barrier()
  7400. +# endif /* CONFIG_SMP */
  7401. +#else
  7402. +# define preempt_disable_rt() barrier()
  7403. +# define preempt_enable_rt() barrier()
  7404. +# define preempt_disable_nort() preempt_disable()
  7405. +# define preempt_enable_nort() preempt_enable()
  7406. +# define migrate_disable() preempt_disable()
  7407. +# define migrate_enable() preempt_enable()
  7408. +#endif
  7409. +
  7410. #ifdef CONFIG_PREEMPT_NOTIFIERS
  7411. struct preempt_notifier;
  7412. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/printk.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/printk.h
  7413. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/printk.h 2017-04-16 10:38:26.000000000 +0200
  7414. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/printk.h 2017-04-18 17:54:26.000000000 +0200
  7415. @@ -126,9 +126,11 @@
  7416. #ifdef CONFIG_EARLY_PRINTK
  7417. extern asmlinkage __printf(1, 2)
  7418. void early_printk(const char *fmt, ...);
  7419. +extern void printk_kill(void);
  7420. #else
  7421. static inline __printf(1, 2) __cold
  7422. void early_printk(const char *s, ...) { }
  7423. +static inline void printk_kill(void) { }
  7424. #endif
  7425. #ifdef CONFIG_PRINTK_NMI
  7426. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/radix-tree.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/radix-tree.h
  7427. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/radix-tree.h 2017-04-16 10:38:26.000000000 +0200
  7428. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/radix-tree.h 2017-04-18 17:54:26.000000000 +0200
  7429. @@ -292,6 +292,8 @@
  7430. int radix_tree_preload(gfp_t gfp_mask);
  7431. int radix_tree_maybe_preload(gfp_t gfp_mask);
  7432. int radix_tree_maybe_preload_order(gfp_t gfp_mask, int order);
  7433. +void radix_tree_preload_end(void);
  7434. +
  7435. void radix_tree_init(void);
  7436. void *radix_tree_tag_set(struct radix_tree_root *root,
  7437. unsigned long index, unsigned int tag);
  7438. @@ -314,11 +316,6 @@
  7439. int radix_tree_tagged(struct radix_tree_root *root, unsigned int tag);
  7440. unsigned long radix_tree_locate_item(struct radix_tree_root *root, void *item);
  7441. -static inline void radix_tree_preload_end(void)
  7442. -{
  7443. - preempt_enable();
  7444. -}
  7445. -
  7446. /**
  7447. * struct radix_tree_iter - radix tree iterator state
  7448. *
  7449. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/random.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/random.h
  7450. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/random.h 2017-04-16 10:38:26.000000000 +0200
  7451. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/random.h 2017-04-18 17:54:26.000000000 +0200
  7452. @@ -31,7 +31,7 @@
  7453. extern void add_input_randomness(unsigned int type, unsigned int code,
  7454. unsigned int value) __latent_entropy;
  7455. -extern void add_interrupt_randomness(int irq, int irq_flags) __latent_entropy;
  7456. +extern void add_interrupt_randomness(int irq, int irq_flags, __u64 ip) __latent_entropy;
  7457. extern void get_random_bytes(void *buf, int nbytes);
  7458. extern int add_random_ready_callback(struct random_ready_callback *rdy);
  7459. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rbtree.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rbtree.h
  7460. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rbtree.h 2017-04-16 10:38:26.000000000 +0200
  7461. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rbtree.h 2017-04-18 17:54:26.000000000 +0200
  7462. @@ -31,7 +31,7 @@
  7463. #include <linux/kernel.h>
  7464. #include <linux/stddef.h>
  7465. -#include <linux/rcupdate.h>
  7466. +#include <linux/rcu_assign_pointer.h>
  7467. struct rb_node {
  7468. unsigned long __rb_parent_color;
  7469. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rbtree_augmented.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rbtree_augmented.h
  7470. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rbtree_augmented.h 2017-04-16 10:38:26.000000000 +0200
  7471. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rbtree_augmented.h 2017-04-18 17:54:26.000000000 +0200
  7472. @@ -26,6 +26,7 @@
  7473. #include <linux/compiler.h>
  7474. #include <linux/rbtree.h>
  7475. +#include <linux/rcupdate.h>
  7476. /*
  7477. * Please note - only struct rb_augment_callbacks and the prototypes for
  7478. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rcu_assign_pointer.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rcu_assign_pointer.h
  7479. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rcu_assign_pointer.h 1970-01-01 01:00:00.000000000 +0100
  7480. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rcu_assign_pointer.h 2017-04-18 17:54:26.000000000 +0200
  7481. @@ -0,0 +1,54 @@
  7482. +#ifndef __LINUX_RCU_ASSIGN_POINTER_H__
  7483. +#define __LINUX_RCU_ASSIGN_POINTER_H__
  7484. +#include <linux/compiler.h>
  7485. +#include <asm/barrier.h>
  7486. +
  7487. +/**
  7488. + * RCU_INITIALIZER() - statically initialize an RCU-protected global variable
  7489. + * @v: The value to statically initialize with.
  7490. + */
  7491. +#define RCU_INITIALIZER(v) (typeof(*(v)) __force __rcu *)(v)
  7492. +
  7493. +/**
  7494. + * rcu_assign_pointer() - assign to RCU-protected pointer
  7495. + * @p: pointer to assign to
  7496. + * @v: value to assign (publish)
  7497. + *
  7498. + * Assigns the specified value to the specified RCU-protected
  7499. + * pointer, ensuring that any concurrent RCU readers will see
  7500. + * any prior initialization.
  7501. + *
  7502. + * Inserts memory barriers on architectures that require them
  7503. + * (which is most of them), and also prevents the compiler from
  7504. + * reordering the code that initializes the structure after the pointer
  7505. + * assignment. More importantly, this call documents which pointers
  7506. + * will be dereferenced by RCU read-side code.
  7507. + *
  7508. + * In some special cases, you may use RCU_INIT_POINTER() instead
  7509. + * of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due
  7510. + * to the fact that it does not constrain either the CPU or the compiler.
  7511. + * That said, using RCU_INIT_POINTER() when you should have used
  7512. + * rcu_assign_pointer() is a very bad thing that results in
  7513. + * impossible-to-diagnose memory corruption. So please be careful.
  7514. + * See the RCU_INIT_POINTER() comment header for details.
  7515. + *
  7516. + * Note that rcu_assign_pointer() evaluates each of its arguments only
  7517. + * once, appearances notwithstanding. One of the "extra" evaluations
  7518. + * is in typeof() and the other visible only to sparse (__CHECKER__),
  7519. + * neither of which actually execute the argument. As with most cpp
  7520. + * macros, this execute-arguments-only-once property is important, so
  7521. + * please be careful when making changes to rcu_assign_pointer() and the
  7522. + * other macros that it invokes.
  7523. + */
  7524. +#define rcu_assign_pointer(p, v) \
  7525. +({ \
  7526. + uintptr_t _r_a_p__v = (uintptr_t)(v); \
  7527. + \
  7528. + if (__builtin_constant_p(v) && (_r_a_p__v) == (uintptr_t)NULL) \
  7529. + WRITE_ONCE((p), (typeof(p))(_r_a_p__v)); \
  7530. + else \
  7531. + smp_store_release(&p, RCU_INITIALIZER((typeof(p))_r_a_p__v)); \
  7532. + _r_a_p__v; \
  7533. +})
  7534. +
  7535. +#endif
  7536. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rcupdate.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rcupdate.h
  7537. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rcupdate.h 2017-04-16 10:38:26.000000000 +0200
  7538. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rcupdate.h 2017-04-18 17:54:26.000000000 +0200
  7539. @@ -46,6 +46,7 @@
  7540. #include <linux/compiler.h>
  7541. #include <linux/ktime.h>
  7542. #include <linux/irqflags.h>
  7543. +#include <linux/rcu_assign_pointer.h>
  7544. #include <asm/barrier.h>
  7545. @@ -178,6 +179,9 @@
  7546. #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
  7547. +#ifdef CONFIG_PREEMPT_RT_FULL
  7548. +#define call_rcu_bh call_rcu
  7549. +#else
  7550. /**
  7551. * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
  7552. * @head: structure to be used for queueing the RCU updates.
  7553. @@ -201,6 +205,7 @@
  7554. */
  7555. void call_rcu_bh(struct rcu_head *head,
  7556. rcu_callback_t func);
  7557. +#endif
  7558. /**
  7559. * call_rcu_sched() - Queue an RCU for invocation after sched grace period.
  7560. @@ -301,6 +306,11 @@
  7561. * types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
  7562. */
  7563. #define rcu_preempt_depth() (current->rcu_read_lock_nesting)
  7564. +#ifndef CONFIG_PREEMPT_RT_FULL
  7565. +#define sched_rcu_preempt_depth() rcu_preempt_depth()
  7566. +#else
  7567. +static inline int sched_rcu_preempt_depth(void) { return 0; }
  7568. +#endif
  7569. #else /* #ifdef CONFIG_PREEMPT_RCU */
  7570. @@ -326,6 +336,8 @@
  7571. return 0;
  7572. }
  7573. +#define sched_rcu_preempt_depth() rcu_preempt_depth()
  7574. +
  7575. #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
  7576. /* Internal to kernel */
  7577. @@ -505,7 +517,14 @@
  7578. int debug_lockdep_rcu_enabled(void);
  7579. int rcu_read_lock_held(void);
  7580. +#ifdef CONFIG_PREEMPT_RT_FULL
  7581. +static inline int rcu_read_lock_bh_held(void)
  7582. +{
  7583. + return rcu_read_lock_held();
  7584. +}
  7585. +#else
  7586. int rcu_read_lock_bh_held(void);
  7587. +#endif
  7588. /**
  7589. * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section?
  7590. @@ -626,54 +645,6 @@
  7591. })
  7592. /**
  7593. - * RCU_INITIALIZER() - statically initialize an RCU-protected global variable
  7594. - * @v: The value to statically initialize with.
  7595. - */
  7596. -#define RCU_INITIALIZER(v) (typeof(*(v)) __force __rcu *)(v)
  7597. -
  7598. -/**
  7599. - * rcu_assign_pointer() - assign to RCU-protected pointer
  7600. - * @p: pointer to assign to
  7601. - * @v: value to assign (publish)
  7602. - *
  7603. - * Assigns the specified value to the specified RCU-protected
  7604. - * pointer, ensuring that any concurrent RCU readers will see
  7605. - * any prior initialization.
  7606. - *
  7607. - * Inserts memory barriers on architectures that require them
  7608. - * (which is most of them), and also prevents the compiler from
  7609. - * reordering the code that initializes the structure after the pointer
  7610. - * assignment. More importantly, this call documents which pointers
  7611. - * will be dereferenced by RCU read-side code.
  7612. - *
  7613. - * In some special cases, you may use RCU_INIT_POINTER() instead
  7614. - * of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due
  7615. - * to the fact that it does not constrain either the CPU or the compiler.
  7616. - * That said, using RCU_INIT_POINTER() when you should have used
  7617. - * rcu_assign_pointer() is a very bad thing that results in
  7618. - * impossible-to-diagnose memory corruption. So please be careful.
  7619. - * See the RCU_INIT_POINTER() comment header for details.
  7620. - *
  7621. - * Note that rcu_assign_pointer() evaluates each of its arguments only
  7622. - * once, appearances notwithstanding. One of the "extra" evaluations
  7623. - * is in typeof() and the other visible only to sparse (__CHECKER__),
  7624. - * neither of which actually execute the argument. As with most cpp
  7625. - * macros, this execute-arguments-only-once property is important, so
  7626. - * please be careful when making changes to rcu_assign_pointer() and the
  7627. - * other macros that it invokes.
  7628. - */
  7629. -#define rcu_assign_pointer(p, v) \
  7630. -({ \
  7631. - uintptr_t _r_a_p__v = (uintptr_t)(v); \
  7632. - \
  7633. - if (__builtin_constant_p(v) && (_r_a_p__v) == (uintptr_t)NULL) \
  7634. - WRITE_ONCE((p), (typeof(p))(_r_a_p__v)); \
  7635. - else \
  7636. - smp_store_release(&p, RCU_INITIALIZER((typeof(p))_r_a_p__v)); \
  7637. - _r_a_p__v; \
  7638. -})
  7639. -
  7640. -/**
  7641. * rcu_access_pointer() - fetch RCU pointer with no dereferencing
  7642. * @p: The pointer to read
  7643. *
  7644. @@ -951,10 +922,14 @@
  7645. static inline void rcu_read_lock_bh(void)
  7646. {
  7647. local_bh_disable();
  7648. +#ifdef CONFIG_PREEMPT_RT_FULL
  7649. + rcu_read_lock();
  7650. +#else
  7651. __acquire(RCU_BH);
  7652. rcu_lock_acquire(&rcu_bh_lock_map);
  7653. RCU_LOCKDEP_WARN(!rcu_is_watching(),
  7654. "rcu_read_lock_bh() used illegally while idle");
  7655. +#endif
  7656. }
  7657. /*
  7658. @@ -964,10 +939,14 @@
  7659. */
  7660. static inline void rcu_read_unlock_bh(void)
  7661. {
  7662. +#ifdef CONFIG_PREEMPT_RT_FULL
  7663. + rcu_read_unlock();
  7664. +#else
  7665. RCU_LOCKDEP_WARN(!rcu_is_watching(),
  7666. "rcu_read_unlock_bh() used illegally while idle");
  7667. rcu_lock_release(&rcu_bh_lock_map);
  7668. __release(RCU_BH);
  7669. +#endif
  7670. local_bh_enable();
  7671. }
  7672. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rcutree.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rcutree.h
  7673. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rcutree.h 2017-04-16 10:38:26.000000000 +0200
  7674. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rcutree.h 2017-04-18 17:54:26.000000000 +0200
  7675. @@ -44,7 +44,11 @@
  7676. rcu_note_context_switch();
  7677. }
  7678. +#ifdef CONFIG_PREEMPT_RT_FULL
  7679. +# define synchronize_rcu_bh synchronize_rcu
  7680. +#else
  7681. void synchronize_rcu_bh(void);
  7682. +#endif
  7683. void synchronize_sched_expedited(void);
  7684. void synchronize_rcu_expedited(void);
  7685. @@ -72,7 +76,11 @@
  7686. }
  7687. void rcu_barrier(void);
  7688. +#ifdef CONFIG_PREEMPT_RT_FULL
  7689. +# define rcu_barrier_bh rcu_barrier
  7690. +#else
  7691. void rcu_barrier_bh(void);
  7692. +#endif
  7693. void rcu_barrier_sched(void);
  7694. unsigned long get_state_synchronize_rcu(void);
  7695. void cond_synchronize_rcu(unsigned long oldstate);
  7696. @@ -82,17 +90,14 @@
  7697. extern unsigned long rcutorture_testseq;
  7698. extern unsigned long rcutorture_vernum;
  7699. unsigned long rcu_batches_started(void);
  7700. -unsigned long rcu_batches_started_bh(void);
  7701. unsigned long rcu_batches_started_sched(void);
  7702. unsigned long rcu_batches_completed(void);
  7703. -unsigned long rcu_batches_completed_bh(void);
  7704. unsigned long rcu_batches_completed_sched(void);
  7705. unsigned long rcu_exp_batches_completed(void);
  7706. unsigned long rcu_exp_batches_completed_sched(void);
  7707. void show_rcu_gp_kthreads(void);
  7708. void rcu_force_quiescent_state(void);
  7709. -void rcu_bh_force_quiescent_state(void);
  7710. void rcu_sched_force_quiescent_state(void);
  7711. void rcu_idle_enter(void);
  7712. @@ -109,6 +114,16 @@
  7713. bool rcu_is_watching(void);
  7714. +#ifndef CONFIG_PREEMPT_RT_FULL
  7715. +void rcu_bh_force_quiescent_state(void);
  7716. +unsigned long rcu_batches_started_bh(void);
  7717. +unsigned long rcu_batches_completed_bh(void);
  7718. +#else
  7719. +# define rcu_bh_force_quiescent_state rcu_force_quiescent_state
  7720. +# define rcu_batches_completed_bh rcu_batches_completed
  7721. +# define rcu_batches_started_bh rcu_batches_completed
  7722. +#endif
  7723. +
  7724. void rcu_all_qs(void);
  7725. /* RCUtree hotplug events */
  7726. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rtmutex.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rtmutex.h
  7727. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rtmutex.h 2017-04-16 10:38:26.000000000 +0200
  7728. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rtmutex.h 2017-04-18 17:54:26.000000000 +0200
  7729. @@ -13,11 +13,15 @@
  7730. #define __LINUX_RT_MUTEX_H
  7731. #include <linux/linkage.h>
  7732. +#include <linux/spinlock_types_raw.h>
  7733. #include <linux/rbtree.h>
  7734. -#include <linux/spinlock_types.h>
  7735. extern int max_lock_depth; /* for sysctl */
  7736. +#ifdef CONFIG_DEBUG_MUTEXES
  7737. +#include <linux/debug_locks.h>
  7738. +#endif
  7739. +
  7740. /**
  7741. * The rt_mutex structure
  7742. *
  7743. @@ -31,8 +35,8 @@
  7744. struct rb_root waiters;
  7745. struct rb_node *waiters_leftmost;
  7746. struct task_struct *owner;
  7747. -#ifdef CONFIG_DEBUG_RT_MUTEXES
  7748. int save_state;
  7749. +#ifdef CONFIG_DEBUG_RT_MUTEXES
  7750. const char *name, *file;
  7751. int line;
  7752. void *magic;
  7753. @@ -55,22 +59,33 @@
  7754. # define rt_mutex_debug_check_no_locks_held(task) do { } while (0)
  7755. #endif
  7756. +# define rt_mutex_init(mutex) \
  7757. + do { \
  7758. + raw_spin_lock_init(&(mutex)->wait_lock); \
  7759. + __rt_mutex_init(mutex, #mutex); \
  7760. + } while (0)
  7761. +
  7762. #ifdef CONFIG_DEBUG_RT_MUTEXES
  7763. # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname) \
  7764. , .name = #mutexname, .file = __FILE__, .line = __LINE__
  7765. -# define rt_mutex_init(mutex) __rt_mutex_init(mutex, __func__)
  7766. extern void rt_mutex_debug_task_free(struct task_struct *tsk);
  7767. #else
  7768. # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname)
  7769. -# define rt_mutex_init(mutex) __rt_mutex_init(mutex, NULL)
  7770. # define rt_mutex_debug_task_free(t) do { } while (0)
  7771. #endif
  7772. -#define __RT_MUTEX_INITIALIZER(mutexname) \
  7773. - { .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \
  7774. +#define __RT_MUTEX_INITIALIZER_PLAIN(mutexname) \
  7775. + .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \
  7776. , .waiters = RB_ROOT \
  7777. , .owner = NULL \
  7778. - __DEBUG_RT_MUTEX_INITIALIZER(mutexname)}
  7779. + __DEBUG_RT_MUTEX_INITIALIZER(mutexname)
  7780. +
  7781. +#define __RT_MUTEX_INITIALIZER(mutexname) \
  7782. + { __RT_MUTEX_INITIALIZER_PLAIN(mutexname) }
  7783. +
  7784. +#define __RT_MUTEX_INITIALIZER_SAVE_STATE(mutexname) \
  7785. + { __RT_MUTEX_INITIALIZER_PLAIN(mutexname) \
  7786. + , .save_state = 1 }
  7787. #define DEFINE_RT_MUTEX(mutexname) \
  7788. struct rt_mutex mutexname = __RT_MUTEX_INITIALIZER(mutexname)
  7789. @@ -90,7 +105,9 @@
  7790. extern void rt_mutex_destroy(struct rt_mutex *lock);
  7791. extern void rt_mutex_lock(struct rt_mutex *lock);
  7792. +extern int rt_mutex_lock_state(struct rt_mutex *lock, int state);
  7793. extern int rt_mutex_lock_interruptible(struct rt_mutex *lock);
  7794. +extern int rt_mutex_lock_killable(struct rt_mutex *lock);
  7795. extern int rt_mutex_timed_lock(struct rt_mutex *lock,
  7796. struct hrtimer_sleeper *timeout);
  7797. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rwlock_rt.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rwlock_rt.h
  7798. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rwlock_rt.h 1970-01-01 01:00:00.000000000 +0100
  7799. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rwlock_rt.h 2017-04-18 17:54:26.000000000 +0200
  7800. @@ -0,0 +1,99 @@
  7801. +#ifndef __LINUX_RWLOCK_RT_H
  7802. +#define __LINUX_RWLOCK_RT_H
  7803. +
  7804. +#ifndef __LINUX_SPINLOCK_H
  7805. +#error Do not include directly. Use spinlock.h
  7806. +#endif
  7807. +
  7808. +#define rwlock_init(rwl) \
  7809. +do { \
  7810. + static struct lock_class_key __key; \
  7811. + \
  7812. + rt_mutex_init(&(rwl)->lock); \
  7813. + __rt_rwlock_init(rwl, #rwl, &__key); \
  7814. +} while (0)
  7815. +
  7816. +extern void __lockfunc rt_write_lock(rwlock_t *rwlock);
  7817. +extern void __lockfunc rt_read_lock(rwlock_t *rwlock);
  7818. +extern int __lockfunc rt_write_trylock(rwlock_t *rwlock);
  7819. +extern int __lockfunc rt_write_trylock_irqsave(rwlock_t *trylock, unsigned long *flags);
  7820. +extern int __lockfunc rt_read_trylock(rwlock_t *rwlock);
  7821. +extern void __lockfunc rt_write_unlock(rwlock_t *rwlock);
  7822. +extern void __lockfunc rt_read_unlock(rwlock_t *rwlock);
  7823. +extern unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock);
  7824. +extern unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock);
  7825. +extern void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key);
  7826. +
  7827. +#define read_trylock(lock) __cond_lock(lock, rt_read_trylock(lock))
  7828. +#define write_trylock(lock) __cond_lock(lock, rt_write_trylock(lock))
  7829. +
  7830. +#define write_trylock_irqsave(lock, flags) \
  7831. + __cond_lock(lock, rt_write_trylock_irqsave(lock, &flags))
  7832. +
  7833. +#define read_lock_irqsave(lock, flags) \
  7834. + do { \
  7835. + typecheck(unsigned long, flags); \
  7836. + flags = rt_read_lock_irqsave(lock); \
  7837. + } while (0)
  7838. +
  7839. +#define write_lock_irqsave(lock, flags) \
  7840. + do { \
  7841. + typecheck(unsigned long, flags); \
  7842. + flags = rt_write_lock_irqsave(lock); \
  7843. + } while (0)
  7844. +
  7845. +#define read_lock(lock) rt_read_lock(lock)
  7846. +
  7847. +#define read_lock_bh(lock) \
  7848. + do { \
  7849. + local_bh_disable(); \
  7850. + rt_read_lock(lock); \
  7851. + } while (0)
  7852. +
  7853. +#define read_lock_irq(lock) read_lock(lock)
  7854. +
  7855. +#define write_lock(lock) rt_write_lock(lock)
  7856. +
  7857. +#define write_lock_bh(lock) \
  7858. + do { \
  7859. + local_bh_disable(); \
  7860. + rt_write_lock(lock); \
  7861. + } while (0)
  7862. +
  7863. +#define write_lock_irq(lock) write_lock(lock)
  7864. +
  7865. +#define read_unlock(lock) rt_read_unlock(lock)
  7866. +
  7867. +#define read_unlock_bh(lock) \
  7868. + do { \
  7869. + rt_read_unlock(lock); \
  7870. + local_bh_enable(); \
  7871. + } while (0)
  7872. +
  7873. +#define read_unlock_irq(lock) read_unlock(lock)
  7874. +
  7875. +#define write_unlock(lock) rt_write_unlock(lock)
  7876. +
  7877. +#define write_unlock_bh(lock) \
  7878. + do { \
  7879. + rt_write_unlock(lock); \
  7880. + local_bh_enable(); \
  7881. + } while (0)
  7882. +
  7883. +#define write_unlock_irq(lock) write_unlock(lock)
  7884. +
  7885. +#define read_unlock_irqrestore(lock, flags) \
  7886. + do { \
  7887. + typecheck(unsigned long, flags); \
  7888. + (void) flags; \
  7889. + rt_read_unlock(lock); \
  7890. + } while (0)
  7891. +
  7892. +#define write_unlock_irqrestore(lock, flags) \
  7893. + do { \
  7894. + typecheck(unsigned long, flags); \
  7895. + (void) flags; \
  7896. + rt_write_unlock(lock); \
  7897. + } while (0)
  7898. +
  7899. +#endif
  7900. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rwlock_types.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rwlock_types.h
  7901. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rwlock_types.h 2017-04-16 10:38:26.000000000 +0200
  7902. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rwlock_types.h 2017-04-18 17:54:26.000000000 +0200
  7903. @@ -1,6 +1,10 @@
  7904. #ifndef __LINUX_RWLOCK_TYPES_H
  7905. #define __LINUX_RWLOCK_TYPES_H
  7906. +#if !defined(__LINUX_SPINLOCK_TYPES_H)
  7907. +# error "Do not include directly, include spinlock_types.h"
  7908. +#endif
  7909. +
  7910. /*
  7911. * include/linux/rwlock_types.h - generic rwlock type definitions
  7912. * and initializers
  7913. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rwlock_types_rt.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rwlock_types_rt.h
  7914. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rwlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100
  7915. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rwlock_types_rt.h 2017-04-18 17:54:26.000000000 +0200
  7916. @@ -0,0 +1,33 @@
  7917. +#ifndef __LINUX_RWLOCK_TYPES_RT_H
  7918. +#define __LINUX_RWLOCK_TYPES_RT_H
  7919. +
  7920. +#ifndef __LINUX_SPINLOCK_TYPES_H
  7921. +#error "Do not include directly. Include spinlock_types.h instead"
  7922. +#endif
  7923. +
  7924. +/*
  7925. + * rwlocks - rtmutex which allows single reader recursion
  7926. + */
  7927. +typedef struct {
  7928. + struct rt_mutex lock;
  7929. + int read_depth;
  7930. + unsigned int break_lock;
  7931. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  7932. + struct lockdep_map dep_map;
  7933. +#endif
  7934. +} rwlock_t;
  7935. +
  7936. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  7937. +# define RW_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname }
  7938. +#else
  7939. +# define RW_DEP_MAP_INIT(lockname)
  7940. +#endif
  7941. +
  7942. +#define __RW_LOCK_UNLOCKED(name) \
  7943. + { .lock = __RT_MUTEX_INITIALIZER_SAVE_STATE(name.lock), \
  7944. + RW_DEP_MAP_INIT(name) }
  7945. +
  7946. +#define DEFINE_RWLOCK(name) \
  7947. + rwlock_t name = __RW_LOCK_UNLOCKED(name)
  7948. +
  7949. +#endif
  7950. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rwsem.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rwsem.h
  7951. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rwsem.h 2017-04-16 10:38:26.000000000 +0200
  7952. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rwsem.h 2017-04-18 17:54:26.000000000 +0200
  7953. @@ -19,6 +19,10 @@
  7954. #include <linux/osq_lock.h>
  7955. #endif
  7956. +#ifdef CONFIG_PREEMPT_RT_FULL
  7957. +#include <linux/rwsem_rt.h>
  7958. +#else /* PREEMPT_RT_FULL */
  7959. +
  7960. struct rw_semaphore;
  7961. #ifdef CONFIG_RWSEM_GENERIC_SPINLOCK
  7962. @@ -106,6 +110,13 @@
  7963. return !list_empty(&sem->wait_list);
  7964. }
  7965. +#endif /* !PREEMPT_RT_FULL */
  7966. +
  7967. +/*
  7968. + * The functions below are the same for all rwsem implementations including
  7969. + * the RT specific variant.
  7970. + */
  7971. +
  7972. /*
  7973. * lock for reading
  7974. */
  7975. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rwsem_rt.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rwsem_rt.h
  7976. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/rwsem_rt.h 1970-01-01 01:00:00.000000000 +0100
  7977. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/rwsem_rt.h 2017-04-18 17:54:26.000000000 +0200
  7978. @@ -0,0 +1,67 @@
  7979. +#ifndef _LINUX_RWSEM_RT_H
  7980. +#define _LINUX_RWSEM_RT_H
  7981. +
  7982. +#ifndef _LINUX_RWSEM_H
  7983. +#error "Include rwsem.h"
  7984. +#endif
  7985. +
  7986. +#include <linux/rtmutex.h>
  7987. +#include <linux/swait.h>
  7988. +
  7989. +#define READER_BIAS (1U << 31)
  7990. +#define WRITER_BIAS (1U << 30)
  7991. +
  7992. +struct rw_semaphore {
  7993. + atomic_t readers;
  7994. + struct rt_mutex rtmutex;
  7995. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  7996. + struct lockdep_map dep_map;
  7997. +#endif
  7998. +};
  7999. +
  8000. +#define __RWSEM_INITIALIZER(name) \
  8001. +{ \
  8002. + .readers = ATOMIC_INIT(READER_BIAS), \
  8003. + .rtmutex = __RT_MUTEX_INITIALIZER(name.rtmutex), \
  8004. + RW_DEP_MAP_INIT(name) \
  8005. +}
  8006. +
  8007. +#define DECLARE_RWSEM(lockname) \
  8008. + struct rw_semaphore lockname = __RWSEM_INITIALIZER(lockname)
  8009. +
  8010. +extern void __rwsem_init(struct rw_semaphore *rwsem, const char *name,
  8011. + struct lock_class_key *key);
  8012. +
  8013. +#define __init_rwsem(sem, name, key) \
  8014. +do { \
  8015. + rt_mutex_init(&(sem)->rtmutex); \
  8016. + __rwsem_init((sem), (name), (key)); \
  8017. +} while (0)
  8018. +
  8019. +#define init_rwsem(sem) \
  8020. +do { \
  8021. + static struct lock_class_key __key; \
  8022. + \
  8023. + __init_rwsem((sem), #sem, &__key); \
  8024. +} while (0)
  8025. +
  8026. +static inline int rwsem_is_locked(struct rw_semaphore *sem)
  8027. +{
  8028. + return atomic_read(&sem->readers) != READER_BIAS;
  8029. +}
  8030. +
  8031. +static inline int rwsem_is_contended(struct rw_semaphore *sem)
  8032. +{
  8033. + return atomic_read(&sem->readers) > 0;
  8034. +}
  8035. +
  8036. +extern void __down_read(struct rw_semaphore *sem);
  8037. +extern int __down_read_trylock(struct rw_semaphore *sem);
  8038. +extern void __down_write(struct rw_semaphore *sem);
  8039. +extern int __must_check __down_write_killable(struct rw_semaphore *sem);
  8040. +extern int __down_write_trylock(struct rw_semaphore *sem);
  8041. +extern void __up_read(struct rw_semaphore *sem);
  8042. +extern void __up_write(struct rw_semaphore *sem);
  8043. +extern void __downgrade_write(struct rw_semaphore *sem);
  8044. +
  8045. +#endif
  8046. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/sched.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/sched.h
  8047. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/sched.h 2017-04-16 10:38:26.000000000 +0200
  8048. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/sched.h 2017-04-18 17:54:26.000000000 +0200
  8049. @@ -26,6 +26,7 @@
  8050. #include <linux/nodemask.h>
  8051. #include <linux/mm_types.h>
  8052. #include <linux/preempt.h>
  8053. +#include <asm/kmap_types.h>
  8054. #include <asm/page.h>
  8055. #include <asm/ptrace.h>
  8056. @@ -243,10 +244,7 @@
  8057. TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
  8058. __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD)
  8059. -#define task_is_traced(task) ((task->state & __TASK_TRACED) != 0)
  8060. #define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0)
  8061. -#define task_is_stopped_or_traced(task) \
  8062. - ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
  8063. #define task_contributes_to_load(task) \
  8064. ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
  8065. (task->flags & PF_FROZEN) == 0 && \
  8066. @@ -312,6 +310,11 @@
  8067. #endif
  8068. +#define __set_current_state_no_track(state_value) \
  8069. + do { current->state = (state_value); } while (0)
  8070. +#define set_current_state_no_track(state_value) \
  8071. + set_mb(current->state, (state_value))
  8072. +
  8073. /* Task command name length */
  8074. #define TASK_COMM_LEN 16
  8075. @@ -1013,8 +1016,18 @@
  8076. struct wake_q_head name = { WAKE_Q_TAIL, &name.first }
  8077. extern void wake_q_add(struct wake_q_head *head,
  8078. - struct task_struct *task);
  8079. -extern void wake_up_q(struct wake_q_head *head);
  8080. + struct task_struct *task);
  8081. +extern void __wake_up_q(struct wake_q_head *head, bool sleeper);
  8082. +
  8083. +static inline void wake_up_q(struct wake_q_head *head)
  8084. +{
  8085. + __wake_up_q(head, false);
  8086. +}
  8087. +
  8088. +static inline void wake_up_q_sleeper(struct wake_q_head *head)
  8089. +{
  8090. + __wake_up_q(head, true);
  8091. +}
  8092. /*
  8093. * sched-domains (multiprocessor balancing) declarations:
  8094. @@ -1481,6 +1494,7 @@
  8095. struct thread_info thread_info;
  8096. #endif
  8097. volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */
  8098. + volatile long saved_state; /* saved state for "spinlock sleepers" */
  8099. void *stack;
  8100. atomic_t usage;
  8101. unsigned int flags; /* per process flags, defined below */
  8102. @@ -1520,6 +1534,12 @@
  8103. #endif
  8104. unsigned int policy;
  8105. +#ifdef CONFIG_PREEMPT_RT_FULL
  8106. + int migrate_disable;
  8107. +# ifdef CONFIG_SCHED_DEBUG
  8108. + int migrate_disable_atomic;
  8109. +# endif
  8110. +#endif
  8111. int nr_cpus_allowed;
  8112. cpumask_t cpus_allowed;
  8113. @@ -1654,6 +1674,9 @@
  8114. struct task_cputime cputime_expires;
  8115. struct list_head cpu_timers[3];
  8116. +#ifdef CONFIG_PREEMPT_RT_BASE
  8117. + struct task_struct *posix_timer_list;
  8118. +#endif
  8119. /* process credentials */
  8120. const struct cred __rcu *ptracer_cred; /* Tracer's credentials at attach */
  8121. @@ -1685,10 +1708,15 @@
  8122. /* signal handlers */
  8123. struct signal_struct *signal;
  8124. struct sighand_struct *sighand;
  8125. + struct sigqueue *sigqueue_cache;
  8126. sigset_t blocked, real_blocked;
  8127. sigset_t saved_sigmask; /* restored if set_restore_sigmask() was used */
  8128. struct sigpending pending;
  8129. +#ifdef CONFIG_PREEMPT_RT_FULL
  8130. + /* TODO: move me into ->restart_block ? */
  8131. + struct siginfo forced_info;
  8132. +#endif
  8133. unsigned long sas_ss_sp;
  8134. size_t sas_ss_size;
  8135. @@ -1917,6 +1945,12 @@
  8136. /* bitmask and counter of trace recursion */
  8137. unsigned long trace_recursion;
  8138. #endif /* CONFIG_TRACING */
  8139. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  8140. + u64 preempt_timestamp_hist;
  8141. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  8142. + long timer_offset;
  8143. +#endif
  8144. +#endif
  8145. #ifdef CONFIG_KCOV
  8146. /* Coverage collection mode enabled for this task (0 if disabled). */
  8147. enum kcov_mode kcov_mode;
  8148. @@ -1942,9 +1976,23 @@
  8149. unsigned int sequential_io;
  8150. unsigned int sequential_io_avg;
  8151. #endif
  8152. +#ifdef CONFIG_PREEMPT_RT_BASE
  8153. + struct rcu_head put_rcu;
  8154. + int softirq_nestcnt;
  8155. + unsigned int softirqs_raised;
  8156. +#endif
  8157. +#ifdef CONFIG_PREEMPT_RT_FULL
  8158. +# if defined CONFIG_HIGHMEM || defined CONFIG_X86_32
  8159. + int kmap_idx;
  8160. + pte_t kmap_pte[KM_TYPE_NR];
  8161. +# endif
  8162. +#endif
  8163. #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
  8164. unsigned long task_state_change;
  8165. #endif
  8166. +#ifdef CONFIG_PREEMPT_RT_FULL
  8167. + int xmit_recursion;
  8168. +#endif
  8169. int pagefault_disabled;
  8170. #ifdef CONFIG_MMU
  8171. struct task_struct *oom_reaper_list;
  8172. @@ -1984,14 +2032,6 @@
  8173. }
  8174. #endif
  8175. -/* Future-safe accessor for struct task_struct's cpus_allowed. */
  8176. -#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
  8177. -
  8178. -static inline int tsk_nr_cpus_allowed(struct task_struct *p)
  8179. -{
  8180. - return p->nr_cpus_allowed;
  8181. -}
  8182. -
  8183. #define TNF_MIGRATED 0x01
  8184. #define TNF_NO_GROUP 0x02
  8185. #define TNF_SHARED 0x04
  8186. @@ -2207,6 +2247,15 @@
  8187. extern void free_task(struct task_struct *tsk);
  8188. #define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0)
  8189. +#ifdef CONFIG_PREEMPT_RT_BASE
  8190. +extern void __put_task_struct_cb(struct rcu_head *rhp);
  8191. +
  8192. +static inline void put_task_struct(struct task_struct *t)
  8193. +{
  8194. + if (atomic_dec_and_test(&t->usage))
  8195. + call_rcu(&t->put_rcu, __put_task_struct_cb);
  8196. +}
  8197. +#else
  8198. extern void __put_task_struct(struct task_struct *t);
  8199. static inline void put_task_struct(struct task_struct *t)
  8200. @@ -2214,6 +2263,7 @@
  8201. if (atomic_dec_and_test(&t->usage))
  8202. __put_task_struct(t);
  8203. }
  8204. +#endif
  8205. struct task_struct *task_rcu_dereference(struct task_struct **ptask);
  8206. struct task_struct *try_get_task_struct(struct task_struct **ptask);
  8207. @@ -2255,6 +2305,7 @@
  8208. /*
  8209. * Per process flags
  8210. */
  8211. +#define PF_IN_SOFTIRQ 0x00000001 /* Task is serving softirq */
  8212. #define PF_EXITING 0x00000004 /* getting shut down */
  8213. #define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */
  8214. #define PF_VCPU 0x00000010 /* I'm a virtual CPU */
  8215. @@ -2423,6 +2474,10 @@
  8216. extern int set_cpus_allowed_ptr(struct task_struct *p,
  8217. const struct cpumask *new_mask);
  8218. +int migrate_me(void);
  8219. +void tell_sched_cpu_down_begin(int cpu);
  8220. +void tell_sched_cpu_down_done(int cpu);
  8221. +
  8222. #else
  8223. static inline void do_set_cpus_allowed(struct task_struct *p,
  8224. const struct cpumask *new_mask)
  8225. @@ -2435,6 +2490,9 @@
  8226. return -EINVAL;
  8227. return 0;
  8228. }
  8229. +static inline int migrate_me(void) { return 0; }
  8230. +static inline void tell_sched_cpu_down_begin(int cpu) { }
  8231. +static inline void tell_sched_cpu_down_done(int cpu) { }
  8232. #endif
  8233. #ifdef CONFIG_NO_HZ_COMMON
  8234. @@ -2673,6 +2731,7 @@
  8235. extern int wake_up_state(struct task_struct *tsk, unsigned int state);
  8236. extern int wake_up_process(struct task_struct *tsk);
  8237. +extern int wake_up_lock_sleeper(struct task_struct * tsk);
  8238. extern void wake_up_new_task(struct task_struct *tsk);
  8239. #ifdef CONFIG_SMP
  8240. extern void kick_process(struct task_struct *tsk);
  8241. @@ -2881,6 +2940,17 @@
  8242. __mmdrop(mm);
  8243. }
  8244. +#ifdef CONFIG_PREEMPT_RT_BASE
  8245. +extern void __mmdrop_delayed(struct rcu_head *rhp);
  8246. +static inline void mmdrop_delayed(struct mm_struct *mm)
  8247. +{
  8248. + if (atomic_dec_and_test(&mm->mm_count))
  8249. + call_rcu(&mm->delayed_drop, __mmdrop_delayed);
  8250. +}
  8251. +#else
  8252. +# define mmdrop_delayed(mm) mmdrop(mm)
  8253. +#endif
  8254. +
  8255. static inline void mmdrop_async_fn(struct work_struct *work)
  8256. {
  8257. struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work);
  8258. @@ -3273,6 +3343,43 @@
  8259. return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
  8260. }
  8261. +#ifdef CONFIG_PREEMPT_LAZY
  8262. +static inline void set_tsk_need_resched_lazy(struct task_struct *tsk)
  8263. +{
  8264. + set_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY);
  8265. +}
  8266. +
  8267. +static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk)
  8268. +{
  8269. + clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY);
  8270. +}
  8271. +
  8272. +static inline int test_tsk_need_resched_lazy(struct task_struct *tsk)
  8273. +{
  8274. + return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY));
  8275. +}
  8276. +
  8277. +static inline int need_resched_lazy(void)
  8278. +{
  8279. + return test_thread_flag(TIF_NEED_RESCHED_LAZY);
  8280. +}
  8281. +
  8282. +static inline int need_resched_now(void)
  8283. +{
  8284. + return test_thread_flag(TIF_NEED_RESCHED);
  8285. +}
  8286. +
  8287. +#else
  8288. +static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk) { }
  8289. +static inline int need_resched_lazy(void) { return 0; }
  8290. +
  8291. +static inline int need_resched_now(void)
  8292. +{
  8293. + return test_thread_flag(TIF_NEED_RESCHED);
  8294. +}
  8295. +
  8296. +#endif
  8297. +
  8298. static inline int restart_syscall(void)
  8299. {
  8300. set_tsk_thread_flag(current, TIF_SIGPENDING);
  8301. @@ -3304,6 +3411,51 @@
  8302. return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p);
  8303. }
  8304. +static inline bool __task_is_stopped_or_traced(struct task_struct *task)
  8305. +{
  8306. + if (task->state & (__TASK_STOPPED | __TASK_TRACED))
  8307. + return true;
  8308. +#ifdef CONFIG_PREEMPT_RT_FULL
  8309. + if (task->saved_state & (__TASK_STOPPED | __TASK_TRACED))
  8310. + return true;
  8311. +#endif
  8312. + return false;
  8313. +}
  8314. +
  8315. +static inline bool task_is_stopped_or_traced(struct task_struct *task)
  8316. +{
  8317. + bool traced_stopped;
  8318. +
  8319. +#ifdef CONFIG_PREEMPT_RT_FULL
  8320. + unsigned long flags;
  8321. +
  8322. + raw_spin_lock_irqsave(&task->pi_lock, flags);
  8323. + traced_stopped = __task_is_stopped_or_traced(task);
  8324. + raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  8325. +#else
  8326. + traced_stopped = __task_is_stopped_or_traced(task);
  8327. +#endif
  8328. + return traced_stopped;
  8329. +}
  8330. +
  8331. +static inline bool task_is_traced(struct task_struct *task)
  8332. +{
  8333. + bool traced = false;
  8334. +
  8335. + if (task->state & __TASK_TRACED)
  8336. + return true;
  8337. +#ifdef CONFIG_PREEMPT_RT_FULL
  8338. + /* in case the task is sleeping on tasklist_lock */
  8339. + raw_spin_lock_irq(&task->pi_lock);
  8340. + if (task->state & __TASK_TRACED)
  8341. + traced = true;
  8342. + else if (task->saved_state & __TASK_TRACED)
  8343. + traced = true;
  8344. + raw_spin_unlock_irq(&task->pi_lock);
  8345. +#endif
  8346. + return traced;
  8347. +}
  8348. +
  8349. /*
  8350. * cond_resched() and cond_resched_lock(): latency reduction via
  8351. * explicit rescheduling in places that are safe. The return
  8352. @@ -3329,12 +3481,16 @@
  8353. __cond_resched_lock(lock); \
  8354. })
  8355. +#ifndef CONFIG_PREEMPT_RT_FULL
  8356. extern int __cond_resched_softirq(void);
  8357. #define cond_resched_softirq() ({ \
  8358. ___might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET); \
  8359. __cond_resched_softirq(); \
  8360. })
  8361. +#else
  8362. +# define cond_resched_softirq() cond_resched()
  8363. +#endif
  8364. static inline void cond_resched_rcu(void)
  8365. {
  8366. @@ -3509,6 +3665,31 @@
  8367. #endif /* CONFIG_SMP */
  8368. +static inline int __migrate_disabled(struct task_struct *p)
  8369. +{
  8370. +#ifdef CONFIG_PREEMPT_RT_FULL
  8371. + return p->migrate_disable;
  8372. +#else
  8373. + return 0;
  8374. +#endif
  8375. +}
  8376. +
  8377. +/* Future-safe accessor for struct task_struct's cpus_allowed. */
  8378. +static inline const struct cpumask *tsk_cpus_allowed(struct task_struct *p)
  8379. +{
  8380. + if (__migrate_disabled(p))
  8381. + return cpumask_of(task_cpu(p));
  8382. +
  8383. + return &p->cpus_allowed;
  8384. +}
  8385. +
  8386. +static inline int tsk_nr_cpus_allowed(struct task_struct *p)
  8387. +{
  8388. + if (__migrate_disabled(p))
  8389. + return 1;
  8390. + return p->nr_cpus_allowed;
  8391. +}
  8392. +
  8393. extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
  8394. extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
  8395. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/seqlock.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/seqlock.h
  8396. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/seqlock.h 2017-04-16 10:38:26.000000000 +0200
  8397. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/seqlock.h 2017-04-18 17:54:26.000000000 +0200
  8398. @@ -220,20 +220,30 @@
  8399. return __read_seqcount_retry(s, start);
  8400. }
  8401. -
  8402. -
  8403. -static inline void raw_write_seqcount_begin(seqcount_t *s)
  8404. +static inline void __raw_write_seqcount_begin(seqcount_t *s)
  8405. {
  8406. s->sequence++;
  8407. smp_wmb();
  8408. }
  8409. -static inline void raw_write_seqcount_end(seqcount_t *s)
  8410. +static inline void raw_write_seqcount_begin(seqcount_t *s)
  8411. +{
  8412. + preempt_disable_rt();
  8413. + __raw_write_seqcount_begin(s);
  8414. +}
  8415. +
  8416. +static inline void __raw_write_seqcount_end(seqcount_t *s)
  8417. {
  8418. smp_wmb();
  8419. s->sequence++;
  8420. }
  8421. +static inline void raw_write_seqcount_end(seqcount_t *s)
  8422. +{
  8423. + __raw_write_seqcount_end(s);
  8424. + preempt_enable_rt();
  8425. +}
  8426. +
  8427. /**
  8428. * raw_write_seqcount_barrier - do a seq write barrier
  8429. * @s: pointer to seqcount_t
  8430. @@ -428,10 +438,32 @@
  8431. /*
  8432. * Read side functions for starting and finalizing a read side section.
  8433. */
  8434. +#ifndef CONFIG_PREEMPT_RT_FULL
  8435. static inline unsigned read_seqbegin(const seqlock_t *sl)
  8436. {
  8437. return read_seqcount_begin(&sl->seqcount);
  8438. }
  8439. +#else
  8440. +/*
  8441. + * Starvation safe read side for RT
  8442. + */
  8443. +static inline unsigned read_seqbegin(seqlock_t *sl)
  8444. +{
  8445. + unsigned ret;
  8446. +
  8447. +repeat:
  8448. + ret = ACCESS_ONCE(sl->seqcount.sequence);
  8449. + if (unlikely(ret & 1)) {
  8450. + /*
  8451. + * Take the lock and let the writer proceed (i.e. evtl
  8452. + * boost it), otherwise we could loop here forever.
  8453. + */
  8454. + spin_unlock_wait(&sl->lock);
  8455. + goto repeat;
  8456. + }
  8457. + return ret;
  8458. +}
  8459. +#endif
  8460. static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start)
  8461. {
  8462. @@ -446,36 +478,45 @@
  8463. static inline void write_seqlock(seqlock_t *sl)
  8464. {
  8465. spin_lock(&sl->lock);
  8466. - write_seqcount_begin(&sl->seqcount);
  8467. + __raw_write_seqcount_begin(&sl->seqcount);
  8468. +}
  8469. +
  8470. +static inline int try_write_seqlock(seqlock_t *sl)
  8471. +{
  8472. + if (spin_trylock(&sl->lock)) {
  8473. + __raw_write_seqcount_begin(&sl->seqcount);
  8474. + return 1;
  8475. + }
  8476. + return 0;
  8477. }
  8478. static inline void write_sequnlock(seqlock_t *sl)
  8479. {
  8480. - write_seqcount_end(&sl->seqcount);
  8481. + __raw_write_seqcount_end(&sl->seqcount);
  8482. spin_unlock(&sl->lock);
  8483. }
  8484. static inline void write_seqlock_bh(seqlock_t *sl)
  8485. {
  8486. spin_lock_bh(&sl->lock);
  8487. - write_seqcount_begin(&sl->seqcount);
  8488. + __raw_write_seqcount_begin(&sl->seqcount);
  8489. }
  8490. static inline void write_sequnlock_bh(seqlock_t *sl)
  8491. {
  8492. - write_seqcount_end(&sl->seqcount);
  8493. + __raw_write_seqcount_end(&sl->seqcount);
  8494. spin_unlock_bh(&sl->lock);
  8495. }
  8496. static inline void write_seqlock_irq(seqlock_t *sl)
  8497. {
  8498. spin_lock_irq(&sl->lock);
  8499. - write_seqcount_begin(&sl->seqcount);
  8500. + __raw_write_seqcount_begin(&sl->seqcount);
  8501. }
  8502. static inline void write_sequnlock_irq(seqlock_t *sl)
  8503. {
  8504. - write_seqcount_end(&sl->seqcount);
  8505. + __raw_write_seqcount_end(&sl->seqcount);
  8506. spin_unlock_irq(&sl->lock);
  8507. }
  8508. @@ -484,7 +525,7 @@
  8509. unsigned long flags;
  8510. spin_lock_irqsave(&sl->lock, flags);
  8511. - write_seqcount_begin(&sl->seqcount);
  8512. + __raw_write_seqcount_begin(&sl->seqcount);
  8513. return flags;
  8514. }
  8515. @@ -494,7 +535,7 @@
  8516. static inline void
  8517. write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags)
  8518. {
  8519. - write_seqcount_end(&sl->seqcount);
  8520. + __raw_write_seqcount_end(&sl->seqcount);
  8521. spin_unlock_irqrestore(&sl->lock, flags);
  8522. }
  8523. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/signal.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/signal.h
  8524. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/signal.h 2017-04-16 10:38:26.000000000 +0200
  8525. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/signal.h 2017-04-18 17:54:26.000000000 +0200
  8526. @@ -233,6 +233,7 @@
  8527. }
  8528. extern void flush_sigqueue(struct sigpending *queue);
  8529. +extern void flush_task_sigqueue(struct task_struct *tsk);
  8530. /* Test if 'sig' is valid signal. Use this instead of testing _NSIG directly */
  8531. static inline int valid_signal(unsigned long sig)
  8532. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/skbuff.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/skbuff.h
  8533. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/skbuff.h 2017-04-16 10:38:26.000000000 +0200
  8534. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/skbuff.h 2017-04-18 17:54:26.000000000 +0200
  8535. @@ -284,6 +284,7 @@
  8536. __u32 qlen;
  8537. spinlock_t lock;
  8538. + raw_spinlock_t raw_lock;
  8539. };
  8540. struct sk_buff;
  8541. @@ -1573,6 +1574,12 @@
  8542. __skb_queue_head_init(list);
  8543. }
  8544. +static inline void skb_queue_head_init_raw(struct sk_buff_head *list)
  8545. +{
  8546. + raw_spin_lock_init(&list->raw_lock);
  8547. + __skb_queue_head_init(list);
  8548. +}
  8549. +
  8550. static inline void skb_queue_head_init_class(struct sk_buff_head *list,
  8551. struct lock_class_key *class)
  8552. {
  8553. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/smp.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/smp.h
  8554. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/smp.h 2017-04-16 10:38:26.000000000 +0200
  8555. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/smp.h 2017-04-18 17:54:26.000000000 +0200
  8556. @@ -120,6 +120,13 @@
  8557. extern void __init setup_nr_cpu_ids(void);
  8558. extern void __init smp_init(void);
  8559. +extern int __boot_cpu_id;
  8560. +
  8561. +static inline int get_boot_cpu_id(void)
  8562. +{
  8563. + return __boot_cpu_id;
  8564. +}
  8565. +
  8566. #else /* !SMP */
  8567. static inline void smp_send_stop(void) { }
  8568. @@ -158,6 +165,11 @@
  8569. static inline void smp_init(void) { }
  8570. #endif
  8571. +static inline int get_boot_cpu_id(void)
  8572. +{
  8573. + return 0;
  8574. +}
  8575. +
  8576. #endif /* !SMP */
  8577. /*
  8578. @@ -185,6 +197,9 @@
  8579. #define get_cpu() ({ preempt_disable(); smp_processor_id(); })
  8580. #define put_cpu() preempt_enable()
  8581. +#define get_cpu_light() ({ migrate_disable(); smp_processor_id(); })
  8582. +#define put_cpu_light() migrate_enable()
  8583. +
  8584. /*
  8585. * Callback to arch code if there's nosmp or maxcpus=0 on the
  8586. * boot command line:
  8587. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock.h
  8588. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock.h 2017-04-16 10:38:26.000000000 +0200
  8589. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock.h 2017-04-18 17:54:26.000000000 +0200
  8590. @@ -271,7 +271,11 @@
  8591. #define raw_spin_can_lock(lock) (!raw_spin_is_locked(lock))
  8592. /* Include rwlock functions */
  8593. -#include <linux/rwlock.h>
  8594. +#ifdef CONFIG_PREEMPT_RT_FULL
  8595. +# include <linux/rwlock_rt.h>
  8596. +#else
  8597. +# include <linux/rwlock.h>
  8598. +#endif
  8599. /*
  8600. * Pull the _spin_*()/_read_*()/_write_*() functions/declarations:
  8601. @@ -282,6 +286,10 @@
  8602. # include <linux/spinlock_api_up.h>
  8603. #endif
  8604. +#ifdef CONFIG_PREEMPT_RT_FULL
  8605. +# include <linux/spinlock_rt.h>
  8606. +#else /* PREEMPT_RT_FULL */
  8607. +
  8608. /*
  8609. * Map the spin_lock functions to the raw variants for PREEMPT_RT=n
  8610. */
  8611. @@ -416,4 +424,6 @@
  8612. #define atomic_dec_and_lock(atomic, lock) \
  8613. __cond_lock(lock, _atomic_dec_and_lock(atomic, lock))
  8614. +#endif /* !PREEMPT_RT_FULL */
  8615. +
  8616. #endif /* __LINUX_SPINLOCK_H */
  8617. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock_api_smp.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock_api_smp.h
  8618. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock_api_smp.h 2017-04-16 10:38:26.000000000 +0200
  8619. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock_api_smp.h 2017-04-18 17:54:26.000000000 +0200
  8620. @@ -189,6 +189,8 @@
  8621. return 0;
  8622. }
  8623. -#include <linux/rwlock_api_smp.h>
  8624. +#ifndef CONFIG_PREEMPT_RT_FULL
  8625. +# include <linux/rwlock_api_smp.h>
  8626. +#endif
  8627. #endif /* __LINUX_SPINLOCK_API_SMP_H */
  8628. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock_rt.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock_rt.h
  8629. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock_rt.h 1970-01-01 01:00:00.000000000 +0100
  8630. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock_rt.h 2017-04-18 17:54:26.000000000 +0200
  8631. @@ -0,0 +1,162 @@
  8632. +#ifndef __LINUX_SPINLOCK_RT_H
  8633. +#define __LINUX_SPINLOCK_RT_H
  8634. +
  8635. +#ifndef __LINUX_SPINLOCK_H
  8636. +#error Do not include directly. Use spinlock.h
  8637. +#endif
  8638. +
  8639. +#include <linux/bug.h>
  8640. +
  8641. +extern void
  8642. +__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key);
  8643. +
  8644. +#define spin_lock_init(slock) \
  8645. +do { \
  8646. + static struct lock_class_key __key; \
  8647. + \
  8648. + rt_mutex_init(&(slock)->lock); \
  8649. + __rt_spin_lock_init(slock, #slock, &__key); \
  8650. +} while (0)
  8651. +
  8652. +void __lockfunc rt_spin_lock__no_mg(spinlock_t *lock);
  8653. +void __lockfunc rt_spin_unlock__no_mg(spinlock_t *lock);
  8654. +int __lockfunc rt_spin_trylock__no_mg(spinlock_t *lock);
  8655. +
  8656. +extern void __lockfunc rt_spin_lock(spinlock_t *lock);
  8657. +extern unsigned long __lockfunc rt_spin_lock_trace_flags(spinlock_t *lock);
  8658. +extern void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass);
  8659. +extern void __lockfunc rt_spin_unlock(spinlock_t *lock);
  8660. +extern void __lockfunc rt_spin_unlock_wait(spinlock_t *lock);
  8661. +extern int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags);
  8662. +extern int __lockfunc rt_spin_trylock_bh(spinlock_t *lock);
  8663. +extern int __lockfunc rt_spin_trylock(spinlock_t *lock);
  8664. +extern int atomic_dec_and_spin_lock(atomic_t *atomic, spinlock_t *lock);
  8665. +
  8666. +/*
  8667. + * lockdep-less calls, for derived types like rwlock:
  8668. + * (for trylock they can use rt_mutex_trylock() directly.
  8669. + */
  8670. +extern void __lockfunc __rt_spin_lock__no_mg(struct rt_mutex *lock);
  8671. +extern void __lockfunc __rt_spin_lock(struct rt_mutex *lock);
  8672. +extern void __lockfunc __rt_spin_unlock(struct rt_mutex *lock);
  8673. +
  8674. +#define spin_lock(lock) rt_spin_lock(lock)
  8675. +
  8676. +#define spin_lock_bh(lock) \
  8677. + do { \
  8678. + local_bh_disable(); \
  8679. + rt_spin_lock(lock); \
  8680. + } while (0)
  8681. +
  8682. +#define spin_lock_irq(lock) spin_lock(lock)
  8683. +
  8684. +#define spin_do_trylock(lock) __cond_lock(lock, rt_spin_trylock(lock))
  8685. +
  8686. +#define spin_trylock(lock) \
  8687. +({ \
  8688. + int __locked; \
  8689. + __locked = spin_do_trylock(lock); \
  8690. + __locked; \
  8691. +})
  8692. +
  8693. +#ifdef CONFIG_LOCKDEP
  8694. +# define spin_lock_nested(lock, subclass) \
  8695. + do { \
  8696. + rt_spin_lock_nested(lock, subclass); \
  8697. + } while (0)
  8698. +
  8699. +#define spin_lock_bh_nested(lock, subclass) \
  8700. + do { \
  8701. + local_bh_disable(); \
  8702. + rt_spin_lock_nested(lock, subclass); \
  8703. + } while (0)
  8704. +
  8705. +# define spin_lock_irqsave_nested(lock, flags, subclass) \
  8706. + do { \
  8707. + typecheck(unsigned long, flags); \
  8708. + flags = 0; \
  8709. + rt_spin_lock_nested(lock, subclass); \
  8710. + } while (0)
  8711. +#else
  8712. +# define spin_lock_nested(lock, subclass) spin_lock(lock)
  8713. +# define spin_lock_bh_nested(lock, subclass) spin_lock_bh(lock)
  8714. +
  8715. +# define spin_lock_irqsave_nested(lock, flags, subclass) \
  8716. + do { \
  8717. + typecheck(unsigned long, flags); \
  8718. + flags = 0; \
  8719. + spin_lock(lock); \
  8720. + } while (0)
  8721. +#endif
  8722. +
  8723. +#define spin_lock_irqsave(lock, flags) \
  8724. + do { \
  8725. + typecheck(unsigned long, flags); \
  8726. + flags = 0; \
  8727. + spin_lock(lock); \
  8728. + } while (0)
  8729. +
  8730. +static inline unsigned long spin_lock_trace_flags(spinlock_t *lock)
  8731. +{
  8732. + unsigned long flags = 0;
  8733. +#ifdef CONFIG_TRACE_IRQFLAGS
  8734. + flags = rt_spin_lock_trace_flags(lock);
  8735. +#else
  8736. + spin_lock(lock); /* lock_local */
  8737. +#endif
  8738. + return flags;
  8739. +}
  8740. +
  8741. +/* FIXME: we need rt_spin_lock_nest_lock */
  8742. +#define spin_lock_nest_lock(lock, nest_lock) spin_lock_nested(lock, 0)
  8743. +
  8744. +#define spin_unlock(lock) rt_spin_unlock(lock)
  8745. +
  8746. +#define spin_unlock_bh(lock) \
  8747. + do { \
  8748. + rt_spin_unlock(lock); \
  8749. + local_bh_enable(); \
  8750. + } while (0)
  8751. +
  8752. +#define spin_unlock_irq(lock) spin_unlock(lock)
  8753. +
  8754. +#define spin_unlock_irqrestore(lock, flags) \
  8755. + do { \
  8756. + typecheck(unsigned long, flags); \
  8757. + (void) flags; \
  8758. + spin_unlock(lock); \
  8759. + } while (0)
  8760. +
  8761. +#define spin_trylock_bh(lock) __cond_lock(lock, rt_spin_trylock_bh(lock))
  8762. +#define spin_trylock_irq(lock) spin_trylock(lock)
  8763. +
  8764. +#define spin_trylock_irqsave(lock, flags) \
  8765. + rt_spin_trylock_irqsave(lock, &(flags))
  8766. +
  8767. +#define spin_unlock_wait(lock) rt_spin_unlock_wait(lock)
  8768. +
  8769. +#ifdef CONFIG_GENERIC_LOCKBREAK
  8770. +# define spin_is_contended(lock) ((lock)->break_lock)
  8771. +#else
  8772. +# define spin_is_contended(lock) (((void)(lock), 0))
  8773. +#endif
  8774. +
  8775. +static inline int spin_can_lock(spinlock_t *lock)
  8776. +{
  8777. + return !rt_mutex_is_locked(&lock->lock);
  8778. +}
  8779. +
  8780. +static inline int spin_is_locked(spinlock_t *lock)
  8781. +{
  8782. + return rt_mutex_is_locked(&lock->lock);
  8783. +}
  8784. +
  8785. +static inline void assert_spin_locked(spinlock_t *lock)
  8786. +{
  8787. + BUG_ON(!spin_is_locked(lock));
  8788. +}
  8789. +
  8790. +#define atomic_dec_and_lock(atomic, lock) \
  8791. + atomic_dec_and_spin_lock(atomic, lock)
  8792. +
  8793. +#endif
  8794. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock_types.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock_types.h
  8795. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock_types.h 2017-04-16 10:38:26.000000000 +0200
  8796. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock_types.h 2017-04-18 17:54:26.000000000 +0200
  8797. @@ -9,80 +9,15 @@
  8798. * Released under the General Public License (GPL).
  8799. */
  8800. -#if defined(CONFIG_SMP)
  8801. -# include <asm/spinlock_types.h>
  8802. -#else
  8803. -# include <linux/spinlock_types_up.h>
  8804. -#endif
  8805. -
  8806. -#include <linux/lockdep.h>
  8807. -
  8808. -typedef struct raw_spinlock {
  8809. - arch_spinlock_t raw_lock;
  8810. -#ifdef CONFIG_GENERIC_LOCKBREAK
  8811. - unsigned int break_lock;
  8812. -#endif
  8813. -#ifdef CONFIG_DEBUG_SPINLOCK
  8814. - unsigned int magic, owner_cpu;
  8815. - void *owner;
  8816. -#endif
  8817. -#ifdef CONFIG_DEBUG_LOCK_ALLOC
  8818. - struct lockdep_map dep_map;
  8819. -#endif
  8820. -} raw_spinlock_t;
  8821. -
  8822. -#define SPINLOCK_MAGIC 0xdead4ead
  8823. -
  8824. -#define SPINLOCK_OWNER_INIT ((void *)-1L)
  8825. -
  8826. -#ifdef CONFIG_DEBUG_LOCK_ALLOC
  8827. -# define SPIN_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname }
  8828. -#else
  8829. -# define SPIN_DEP_MAP_INIT(lockname)
  8830. -#endif
  8831. +#include <linux/spinlock_types_raw.h>
  8832. -#ifdef CONFIG_DEBUG_SPINLOCK
  8833. -# define SPIN_DEBUG_INIT(lockname) \
  8834. - .magic = SPINLOCK_MAGIC, \
  8835. - .owner_cpu = -1, \
  8836. - .owner = SPINLOCK_OWNER_INIT,
  8837. +#ifndef CONFIG_PREEMPT_RT_FULL
  8838. +# include <linux/spinlock_types_nort.h>
  8839. +# include <linux/rwlock_types.h>
  8840. #else
  8841. -# define SPIN_DEBUG_INIT(lockname)
  8842. +# include <linux/rtmutex.h>
  8843. +# include <linux/spinlock_types_rt.h>
  8844. +# include <linux/rwlock_types_rt.h>
  8845. #endif
  8846. -#define __RAW_SPIN_LOCK_INITIALIZER(lockname) \
  8847. - { \
  8848. - .raw_lock = __ARCH_SPIN_LOCK_UNLOCKED, \
  8849. - SPIN_DEBUG_INIT(lockname) \
  8850. - SPIN_DEP_MAP_INIT(lockname) }
  8851. -
  8852. -#define __RAW_SPIN_LOCK_UNLOCKED(lockname) \
  8853. - (raw_spinlock_t) __RAW_SPIN_LOCK_INITIALIZER(lockname)
  8854. -
  8855. -#define DEFINE_RAW_SPINLOCK(x) raw_spinlock_t x = __RAW_SPIN_LOCK_UNLOCKED(x)
  8856. -
  8857. -typedef struct spinlock {
  8858. - union {
  8859. - struct raw_spinlock rlock;
  8860. -
  8861. -#ifdef CONFIG_DEBUG_LOCK_ALLOC
  8862. -# define LOCK_PADSIZE (offsetof(struct raw_spinlock, dep_map))
  8863. - struct {
  8864. - u8 __padding[LOCK_PADSIZE];
  8865. - struct lockdep_map dep_map;
  8866. - };
  8867. -#endif
  8868. - };
  8869. -} spinlock_t;
  8870. -
  8871. -#define __SPIN_LOCK_INITIALIZER(lockname) \
  8872. - { { .rlock = __RAW_SPIN_LOCK_INITIALIZER(lockname) } }
  8873. -
  8874. -#define __SPIN_LOCK_UNLOCKED(lockname) \
  8875. - (spinlock_t ) __SPIN_LOCK_INITIALIZER(lockname)
  8876. -
  8877. -#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x)
  8878. -
  8879. -#include <linux/rwlock_types.h>
  8880. -
  8881. #endif /* __LINUX_SPINLOCK_TYPES_H */
  8882. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock_types_nort.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock_types_nort.h
  8883. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock_types_nort.h 1970-01-01 01:00:00.000000000 +0100
  8884. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock_types_nort.h 2017-04-18 17:54:26.000000000 +0200
  8885. @@ -0,0 +1,33 @@
  8886. +#ifndef __LINUX_SPINLOCK_TYPES_NORT_H
  8887. +#define __LINUX_SPINLOCK_TYPES_NORT_H
  8888. +
  8889. +#ifndef __LINUX_SPINLOCK_TYPES_H
  8890. +#error "Do not include directly. Include spinlock_types.h instead"
  8891. +#endif
  8892. +
  8893. +/*
  8894. + * The non RT version maps spinlocks to raw_spinlocks
  8895. + */
  8896. +typedef struct spinlock {
  8897. + union {
  8898. + struct raw_spinlock rlock;
  8899. +
  8900. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  8901. +# define LOCK_PADSIZE (offsetof(struct raw_spinlock, dep_map))
  8902. + struct {
  8903. + u8 __padding[LOCK_PADSIZE];
  8904. + struct lockdep_map dep_map;
  8905. + };
  8906. +#endif
  8907. + };
  8908. +} spinlock_t;
  8909. +
  8910. +#define __SPIN_LOCK_INITIALIZER(lockname) \
  8911. + { { .rlock = __RAW_SPIN_LOCK_INITIALIZER(lockname) } }
  8912. +
  8913. +#define __SPIN_LOCK_UNLOCKED(lockname) \
  8914. + (spinlock_t ) __SPIN_LOCK_INITIALIZER(lockname)
  8915. +
  8916. +#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x)
  8917. +
  8918. +#endif
  8919. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock_types_raw.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock_types_raw.h
  8920. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock_types_raw.h 1970-01-01 01:00:00.000000000 +0100
  8921. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock_types_raw.h 2017-04-18 17:54:26.000000000 +0200
  8922. @@ -0,0 +1,56 @@
  8923. +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H
  8924. +#define __LINUX_SPINLOCK_TYPES_RAW_H
  8925. +
  8926. +#if defined(CONFIG_SMP)
  8927. +# include <asm/spinlock_types.h>
  8928. +#else
  8929. +# include <linux/spinlock_types_up.h>
  8930. +#endif
  8931. +
  8932. +#include <linux/lockdep.h>
  8933. +
  8934. +typedef struct raw_spinlock {
  8935. + arch_spinlock_t raw_lock;
  8936. +#ifdef CONFIG_GENERIC_LOCKBREAK
  8937. + unsigned int break_lock;
  8938. +#endif
  8939. +#ifdef CONFIG_DEBUG_SPINLOCK
  8940. + unsigned int magic, owner_cpu;
  8941. + void *owner;
  8942. +#endif
  8943. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  8944. + struct lockdep_map dep_map;
  8945. +#endif
  8946. +} raw_spinlock_t;
  8947. +
  8948. +#define SPINLOCK_MAGIC 0xdead4ead
  8949. +
  8950. +#define SPINLOCK_OWNER_INIT ((void *)-1L)
  8951. +
  8952. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  8953. +# define SPIN_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname }
  8954. +#else
  8955. +# define SPIN_DEP_MAP_INIT(lockname)
  8956. +#endif
  8957. +
  8958. +#ifdef CONFIG_DEBUG_SPINLOCK
  8959. +# define SPIN_DEBUG_INIT(lockname) \
  8960. + .magic = SPINLOCK_MAGIC, \
  8961. + .owner_cpu = -1, \
  8962. + .owner = SPINLOCK_OWNER_INIT,
  8963. +#else
  8964. +# define SPIN_DEBUG_INIT(lockname)
  8965. +#endif
  8966. +
  8967. +#define __RAW_SPIN_LOCK_INITIALIZER(lockname) \
  8968. + { \
  8969. + .raw_lock = __ARCH_SPIN_LOCK_UNLOCKED, \
  8970. + SPIN_DEBUG_INIT(lockname) \
  8971. + SPIN_DEP_MAP_INIT(lockname) }
  8972. +
  8973. +#define __RAW_SPIN_LOCK_UNLOCKED(lockname) \
  8974. + (raw_spinlock_t) __RAW_SPIN_LOCK_INITIALIZER(lockname)
  8975. +
  8976. +#define DEFINE_RAW_SPINLOCK(x) raw_spinlock_t x = __RAW_SPIN_LOCK_UNLOCKED(x)
  8977. +
  8978. +#endif
  8979. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock_types_rt.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock_types_rt.h
  8980. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/spinlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100
  8981. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/spinlock_types_rt.h 2017-04-18 17:54:26.000000000 +0200
  8982. @@ -0,0 +1,48 @@
  8983. +#ifndef __LINUX_SPINLOCK_TYPES_RT_H
  8984. +#define __LINUX_SPINLOCK_TYPES_RT_H
  8985. +
  8986. +#ifndef __LINUX_SPINLOCK_TYPES_H
  8987. +#error "Do not include directly. Include spinlock_types.h instead"
  8988. +#endif
  8989. +
  8990. +#include <linux/cache.h>
  8991. +
  8992. +/*
  8993. + * PREEMPT_RT: spinlocks - an RT mutex plus lock-break field:
  8994. + */
  8995. +typedef struct spinlock {
  8996. + struct rt_mutex lock;
  8997. + unsigned int break_lock;
  8998. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  8999. + struct lockdep_map dep_map;
  9000. +#endif
  9001. +} spinlock_t;
  9002. +
  9003. +#ifdef CONFIG_DEBUG_RT_MUTEXES
  9004. +# define __RT_SPIN_INITIALIZER(name) \
  9005. + { \
  9006. + .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \
  9007. + .save_state = 1, \
  9008. + .file = __FILE__, \
  9009. + .line = __LINE__ , \
  9010. + }
  9011. +#else
  9012. +# define __RT_SPIN_INITIALIZER(name) \
  9013. + { \
  9014. + .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \
  9015. + .save_state = 1, \
  9016. + }
  9017. +#endif
  9018. +
  9019. +/*
  9020. +.wait_list = PLIST_HEAD_INIT_RAW((name).lock.wait_list, (name).lock.wait_lock)
  9021. +*/
  9022. +
  9023. +#define __SPIN_LOCK_UNLOCKED(name) \
  9024. + { .lock = __RT_SPIN_INITIALIZER(name.lock), \
  9025. + SPIN_DEP_MAP_INIT(name) }
  9026. +
  9027. +#define DEFINE_SPINLOCK(name) \
  9028. + spinlock_t name = __SPIN_LOCK_UNLOCKED(name)
  9029. +
  9030. +#endif
  9031. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/srcu.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/srcu.h
  9032. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/srcu.h 2017-04-16 10:38:26.000000000 +0200
  9033. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/srcu.h 2017-04-18 17:54:26.000000000 +0200
  9034. @@ -84,10 +84,10 @@
  9035. void process_srcu(struct work_struct *work);
  9036. -#define __SRCU_STRUCT_INIT(name) \
  9037. +#define __SRCU_STRUCT_INIT(name, pcpu_name) \
  9038. { \
  9039. .completed = -300, \
  9040. - .per_cpu_ref = &name##_srcu_array, \
  9041. + .per_cpu_ref = &pcpu_name, \
  9042. .queue_lock = __SPIN_LOCK_UNLOCKED(name.queue_lock), \
  9043. .running = false, \
  9044. .batch_queue = RCU_BATCH_INIT(name.batch_queue), \
  9045. @@ -119,7 +119,7 @@
  9046. */
  9047. #define __DEFINE_SRCU(name, is_static) \
  9048. static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\
  9049. - is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name)
  9050. + is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name##_srcu_array)
  9051. #define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */)
  9052. #define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static)
  9053. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/suspend.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/suspend.h
  9054. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/suspend.h 2017-04-16 10:38:26.000000000 +0200
  9055. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/suspend.h 2017-04-18 17:54:26.000000000 +0200
  9056. @@ -193,6 +193,12 @@
  9057. void (*end)(void);
  9058. };
  9059. +#if defined(CONFIG_SUSPEND) || defined(CONFIG_HIBERNATION)
  9060. +extern bool pm_in_action;
  9061. +#else
  9062. +# define pm_in_action false
  9063. +#endif
  9064. +
  9065. #ifdef CONFIG_SUSPEND
  9066. /**
  9067. * suspend_set_ops - set platform dependent suspend operations
  9068. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/swait.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/swait.h
  9069. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/swait.h 2017-04-16 10:38:26.000000000 +0200
  9070. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/swait.h 2017-04-18 17:54:26.000000000 +0200
  9071. @@ -87,6 +87,7 @@
  9072. extern void swake_up(struct swait_queue_head *q);
  9073. extern void swake_up_all(struct swait_queue_head *q);
  9074. extern void swake_up_locked(struct swait_queue_head *q);
  9075. +extern void swake_up_all_locked(struct swait_queue_head *q);
  9076. extern void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait);
  9077. extern void prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait, int state);
  9078. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/swap.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/swap.h
  9079. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/swap.h 2017-04-16 10:38:26.000000000 +0200
  9080. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/swap.h 2017-04-18 17:54:26.000000000 +0200
  9081. @@ -11,6 +11,7 @@
  9082. #include <linux/fs.h>
  9083. #include <linux/atomic.h>
  9084. #include <linux/page-flags.h>
  9085. +#include <linux/locallock.h>
  9086. #include <asm/page.h>
  9087. struct notifier_block;
  9088. @@ -247,7 +248,8 @@
  9089. void *workingset_eviction(struct address_space *mapping, struct page *page);
  9090. bool workingset_refault(void *shadow);
  9091. void workingset_activation(struct page *page);
  9092. -extern struct list_lru workingset_shadow_nodes;
  9093. +extern struct list_lru __workingset_shadow_nodes;
  9094. +DECLARE_LOCAL_IRQ_LOCK(workingset_shadow_lock);
  9095. static inline unsigned int workingset_node_pages(struct radix_tree_node *node)
  9096. {
  9097. @@ -292,6 +294,7 @@
  9098. /* linux/mm/swap.c */
  9099. +DECLARE_LOCAL_IRQ_LOCK(swapvec_lock);
  9100. extern void lru_cache_add(struct page *);
  9101. extern void lru_cache_add_anon(struct page *page);
  9102. extern void lru_cache_add_file(struct page *page);
  9103. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/swork.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/swork.h
  9104. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/swork.h 1970-01-01 01:00:00.000000000 +0100
  9105. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/swork.h 2017-04-18 17:54:26.000000000 +0200
  9106. @@ -0,0 +1,24 @@
  9107. +#ifndef _LINUX_SWORK_H
  9108. +#define _LINUX_SWORK_H
  9109. +
  9110. +#include <linux/list.h>
  9111. +
  9112. +struct swork_event {
  9113. + struct list_head item;
  9114. + unsigned long flags;
  9115. + void (*func)(struct swork_event *);
  9116. +};
  9117. +
  9118. +static inline void INIT_SWORK(struct swork_event *event,
  9119. + void (*func)(struct swork_event *))
  9120. +{
  9121. + event->flags = 0;
  9122. + event->func = func;
  9123. +}
  9124. +
  9125. +bool swork_queue(struct swork_event *sev);
  9126. +
  9127. +int swork_get(void);
  9128. +void swork_put(void);
  9129. +
  9130. +#endif /* _LINUX_SWORK_H */
  9131. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/thread_info.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/thread_info.h
  9132. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/thread_info.h 2017-04-16 10:38:26.000000000 +0200
  9133. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/thread_info.h 2017-04-18 17:54:26.000000000 +0200
  9134. @@ -107,7 +107,17 @@
  9135. #define test_thread_flag(flag) \
  9136. test_ti_thread_flag(current_thread_info(), flag)
  9137. -#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED)
  9138. +#ifdef CONFIG_PREEMPT_LAZY
  9139. +#define tif_need_resched() (test_thread_flag(TIF_NEED_RESCHED) || \
  9140. + test_thread_flag(TIF_NEED_RESCHED_LAZY))
  9141. +#define tif_need_resched_now() (test_thread_flag(TIF_NEED_RESCHED))
  9142. +#define tif_need_resched_lazy() test_thread_flag(TIF_NEED_RESCHED_LAZY))
  9143. +
  9144. +#else
  9145. +#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED)
  9146. +#define tif_need_resched_now() test_thread_flag(TIF_NEED_RESCHED)
  9147. +#define tif_need_resched_lazy() 0
  9148. +#endif
  9149. #ifndef CONFIG_HAVE_ARCH_WITHIN_STACK_FRAMES
  9150. static inline int arch_within_stack_frames(const void * const stack,
  9151. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/timer.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/timer.h
  9152. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/timer.h 2017-04-16 10:38:26.000000000 +0200
  9153. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/timer.h 2017-04-18 17:54:26.000000000 +0200
  9154. @@ -241,7 +241,7 @@
  9155. extern int try_to_del_timer_sync(struct timer_list *timer);
  9156. -#ifdef CONFIG_SMP
  9157. +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
  9158. extern int del_timer_sync(struct timer_list *timer);
  9159. #else
  9160. # define del_timer_sync(t) del_timer(t)
  9161. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/trace_events.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/trace_events.h
  9162. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/trace_events.h 2017-04-16 10:38:26.000000000 +0200
  9163. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/trace_events.h 2017-04-18 17:54:26.000000000 +0200
  9164. @@ -56,6 +56,9 @@
  9165. unsigned char flags;
  9166. unsigned char preempt_count;
  9167. int pid;
  9168. + unsigned short migrate_disable;
  9169. + unsigned short padding;
  9170. + unsigned char preempt_lazy_count;
  9171. };
  9172. #define TRACE_EVENT_TYPE_MAX \
  9173. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/uaccess.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/uaccess.h
  9174. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/uaccess.h 2017-04-16 10:38:26.000000000 +0200
  9175. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/uaccess.h 2017-04-18 17:54:26.000000000 +0200
  9176. @@ -24,6 +24,7 @@
  9177. */
  9178. static inline void pagefault_disable(void)
  9179. {
  9180. + migrate_disable();
  9181. pagefault_disabled_inc();
  9182. /*
  9183. * make sure to have issued the store before a pagefault
  9184. @@ -40,6 +41,7 @@
  9185. */
  9186. barrier();
  9187. pagefault_disabled_dec();
  9188. + migrate_enable();
  9189. }
  9190. /*
  9191. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/uprobes.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/uprobes.h
  9192. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/uprobes.h 2017-04-16 10:38:26.000000000 +0200
  9193. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/uprobes.h 2017-04-18 17:54:26.000000000 +0200
  9194. @@ -27,6 +27,7 @@
  9195. #include <linux/errno.h>
  9196. #include <linux/rbtree.h>
  9197. #include <linux/types.h>
  9198. +#include <linux/wait.h>
  9199. struct vm_area_struct;
  9200. struct mm_struct;
  9201. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/vmstat.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/vmstat.h
  9202. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/vmstat.h 2017-04-16 10:38:27.000000000 +0200
  9203. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/vmstat.h 2017-04-18 17:54:26.000000000 +0200
  9204. @@ -33,7 +33,9 @@
  9205. */
  9206. static inline void __count_vm_event(enum vm_event_item item)
  9207. {
  9208. + preempt_disable_rt();
  9209. raw_cpu_inc(vm_event_states.event[item]);
  9210. + preempt_enable_rt();
  9211. }
  9212. static inline void count_vm_event(enum vm_event_item item)
  9213. @@ -43,7 +45,9 @@
  9214. static inline void __count_vm_events(enum vm_event_item item, long delta)
  9215. {
  9216. + preempt_disable_rt();
  9217. raw_cpu_add(vm_event_states.event[item], delta);
  9218. + preempt_enable_rt();
  9219. }
  9220. static inline void count_vm_events(enum vm_event_item item, long delta)
  9221. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/wait.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/wait.h
  9222. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/linux/wait.h 2017-04-16 10:38:27.000000000 +0200
  9223. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/linux/wait.h 2017-04-18 17:54:26.000000000 +0200
  9224. @@ -8,6 +8,7 @@
  9225. #include <linux/spinlock.h>
  9226. #include <asm/current.h>
  9227. #include <uapi/linux/wait.h>
  9228. +#include <linux/atomic.h>
  9229. typedef struct __wait_queue wait_queue_t;
  9230. typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key);
  9231. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/net/dst.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/net/dst.h
  9232. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/net/dst.h 2017-04-16 10:38:27.000000000 +0200
  9233. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/net/dst.h 2017-04-18 17:54:26.000000000 +0200
  9234. @@ -446,7 +446,7 @@
  9235. static inline int dst_neigh_output(struct dst_entry *dst, struct neighbour *n,
  9236. struct sk_buff *skb)
  9237. {
  9238. - const struct hh_cache *hh;
  9239. + struct hh_cache *hh;
  9240. if (dst->pending_confirm) {
  9241. unsigned long now = jiffies;
  9242. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/net/gen_stats.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/net/gen_stats.h
  9243. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/net/gen_stats.h 2017-04-16 10:38:27.000000000 +0200
  9244. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/net/gen_stats.h 2017-04-18 17:54:26.000000000 +0200
  9245. @@ -5,6 +5,7 @@
  9246. #include <linux/socket.h>
  9247. #include <linux/rtnetlink.h>
  9248. #include <linux/pkt_sched.h>
  9249. +#include <net/net_seq_lock.h>
  9250. struct gnet_stats_basic_cpu {
  9251. struct gnet_stats_basic_packed bstats;
  9252. @@ -33,11 +34,11 @@
  9253. spinlock_t *lock, struct gnet_dump *d,
  9254. int padattr);
  9255. -int gnet_stats_copy_basic(const seqcount_t *running,
  9256. +int gnet_stats_copy_basic(net_seqlock_t *running,
  9257. struct gnet_dump *d,
  9258. struct gnet_stats_basic_cpu __percpu *cpu,
  9259. struct gnet_stats_basic_packed *b);
  9260. -void __gnet_stats_copy_basic(const seqcount_t *running,
  9261. +void __gnet_stats_copy_basic(net_seqlock_t *running,
  9262. struct gnet_stats_basic_packed *bstats,
  9263. struct gnet_stats_basic_cpu __percpu *cpu,
  9264. struct gnet_stats_basic_packed *b);
  9265. @@ -55,14 +56,14 @@
  9266. struct gnet_stats_basic_cpu __percpu *cpu_bstats,
  9267. struct gnet_stats_rate_est64 *rate_est,
  9268. spinlock_t *stats_lock,
  9269. - seqcount_t *running, struct nlattr *opt);
  9270. + net_seqlock_t *running, struct nlattr *opt);
  9271. void gen_kill_estimator(struct gnet_stats_basic_packed *bstats,
  9272. struct gnet_stats_rate_est64 *rate_est);
  9273. int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
  9274. struct gnet_stats_basic_cpu __percpu *cpu_bstats,
  9275. struct gnet_stats_rate_est64 *rate_est,
  9276. spinlock_t *stats_lock,
  9277. - seqcount_t *running, struct nlattr *opt);
  9278. + net_seqlock_t *running, struct nlattr *opt);
  9279. bool gen_estimator_active(const struct gnet_stats_basic_packed *bstats,
  9280. const struct gnet_stats_rate_est64 *rate_est);
  9281. #endif
  9282. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/net/neighbour.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/net/neighbour.h
  9283. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/net/neighbour.h 2017-04-16 10:38:27.000000000 +0200
  9284. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/net/neighbour.h 2017-04-18 17:54:26.000000000 +0200
  9285. @@ -446,7 +446,7 @@
  9286. }
  9287. #endif
  9288. -static inline int neigh_hh_output(const struct hh_cache *hh, struct sk_buff *skb)
  9289. +static inline int neigh_hh_output(struct hh_cache *hh, struct sk_buff *skb)
  9290. {
  9291. unsigned int seq;
  9292. int hh_len;
  9293. @@ -501,7 +501,7 @@
  9294. #define NEIGH_CB(skb) ((struct neighbour_cb *)(skb)->cb)
  9295. -static inline void neigh_ha_snapshot(char *dst, const struct neighbour *n,
  9296. +static inline void neigh_ha_snapshot(char *dst, struct neighbour *n,
  9297. const struct net_device *dev)
  9298. {
  9299. unsigned int seq;
  9300. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/net/net_seq_lock.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/net/net_seq_lock.h
  9301. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/net/net_seq_lock.h 1970-01-01 01:00:00.000000000 +0100
  9302. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/net/net_seq_lock.h 2017-04-18 17:54:26.000000000 +0200
  9303. @@ -0,0 +1,15 @@
  9304. +#ifndef __NET_NET_SEQ_LOCK_H__
  9305. +#define __NET_NET_SEQ_LOCK_H__
  9306. +
  9307. +#ifdef CONFIG_PREEMPT_RT_BASE
  9308. +# define net_seqlock_t seqlock_t
  9309. +# define net_seq_begin(__r) read_seqbegin(__r)
  9310. +# define net_seq_retry(__r, __s) read_seqretry(__r, __s)
  9311. +
  9312. +#else
  9313. +# define net_seqlock_t seqcount_t
  9314. +# define net_seq_begin(__r) read_seqcount_begin(__r)
  9315. +# define net_seq_retry(__r, __s) read_seqcount_retry(__r, __s)
  9316. +#endif
  9317. +
  9318. +#endif
  9319. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/net/netns/ipv4.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/net/netns/ipv4.h
  9320. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/net/netns/ipv4.h 2017-04-16 10:38:27.000000000 +0200
  9321. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/net/netns/ipv4.h 2017-04-18 17:54:26.000000000 +0200
  9322. @@ -69,6 +69,7 @@
  9323. int sysctl_icmp_echo_ignore_all;
  9324. int sysctl_icmp_echo_ignore_broadcasts;
  9325. + int sysctl_icmp_echo_sysrq;
  9326. int sysctl_icmp_ignore_bogus_error_responses;
  9327. int sysctl_icmp_ratelimit;
  9328. int sysctl_icmp_ratemask;
  9329. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/net/sch_generic.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/net/sch_generic.h
  9330. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/net/sch_generic.h 2017-04-16 10:38:27.000000000 +0200
  9331. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/net/sch_generic.h 2017-04-18 17:54:26.000000000 +0200
  9332. @@ -10,6 +10,7 @@
  9333. #include <linux/dynamic_queue_limits.h>
  9334. #include <net/gen_stats.h>
  9335. #include <net/rtnetlink.h>
  9336. +#include <net/net_seq_lock.h>
  9337. struct Qdisc_ops;
  9338. struct qdisc_walker;
  9339. @@ -86,7 +87,7 @@
  9340. struct sk_buff *gso_skb ____cacheline_aligned_in_smp;
  9341. struct qdisc_skb_head q;
  9342. struct gnet_stats_basic_packed bstats;
  9343. - seqcount_t running;
  9344. + net_seqlock_t running;
  9345. struct gnet_stats_queue qstats;
  9346. unsigned long state;
  9347. struct Qdisc *next_sched;
  9348. @@ -98,13 +99,22 @@
  9349. spinlock_t busylock ____cacheline_aligned_in_smp;
  9350. };
  9351. -static inline bool qdisc_is_running(const struct Qdisc *qdisc)
  9352. +static inline bool qdisc_is_running(struct Qdisc *qdisc)
  9353. {
  9354. +#ifdef CONFIG_PREEMPT_RT_BASE
  9355. + return spin_is_locked(&qdisc->running.lock) ? true : false;
  9356. +#else
  9357. return (raw_read_seqcount(&qdisc->running) & 1) ? true : false;
  9358. +#endif
  9359. }
  9360. static inline bool qdisc_run_begin(struct Qdisc *qdisc)
  9361. {
  9362. +#ifdef CONFIG_PREEMPT_RT_BASE
  9363. + if (try_write_seqlock(&qdisc->running))
  9364. + return true;
  9365. + return false;
  9366. +#else
  9367. if (qdisc_is_running(qdisc))
  9368. return false;
  9369. /* Variant of write_seqcount_begin() telling lockdep a trylock
  9370. @@ -113,11 +123,16 @@
  9371. raw_write_seqcount_begin(&qdisc->running);
  9372. seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_);
  9373. return true;
  9374. +#endif
  9375. }
  9376. static inline void qdisc_run_end(struct Qdisc *qdisc)
  9377. {
  9378. +#ifdef CONFIG_PREEMPT_RT_BASE
  9379. + write_sequnlock(&qdisc->running);
  9380. +#else
  9381. write_seqcount_end(&qdisc->running);
  9382. +#endif
  9383. }
  9384. static inline bool qdisc_may_bulk(const struct Qdisc *qdisc)
  9385. @@ -308,7 +323,7 @@
  9386. return qdisc_lock(root);
  9387. }
  9388. -static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc)
  9389. +static inline net_seqlock_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc)
  9390. {
  9391. struct Qdisc *root = qdisc_root_sleeping(qdisc);
  9392. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/trace/events/hist.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/trace/events/hist.h
  9393. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/trace/events/hist.h 1970-01-01 01:00:00.000000000 +0100
  9394. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/trace/events/hist.h 2017-04-18 17:54:26.000000000 +0200
  9395. @@ -0,0 +1,73 @@
  9396. +#undef TRACE_SYSTEM
  9397. +#define TRACE_SYSTEM hist
  9398. +
  9399. +#if !defined(_TRACE_HIST_H) || defined(TRACE_HEADER_MULTI_READ)
  9400. +#define _TRACE_HIST_H
  9401. +
  9402. +#include "latency_hist.h"
  9403. +#include <linux/tracepoint.h>
  9404. +
  9405. +#if !defined(CONFIG_PREEMPT_OFF_HIST) && !defined(CONFIG_INTERRUPT_OFF_HIST)
  9406. +#define trace_preemptirqsoff_hist(a, b)
  9407. +#define trace_preemptirqsoff_hist_rcuidle(a, b)
  9408. +#else
  9409. +TRACE_EVENT(preemptirqsoff_hist,
  9410. +
  9411. + TP_PROTO(int reason, int starthist),
  9412. +
  9413. + TP_ARGS(reason, starthist),
  9414. +
  9415. + TP_STRUCT__entry(
  9416. + __field(int, reason)
  9417. + __field(int, starthist)
  9418. + ),
  9419. +
  9420. + TP_fast_assign(
  9421. + __entry->reason = reason;
  9422. + __entry->starthist = starthist;
  9423. + ),
  9424. +
  9425. + TP_printk("reason=%s starthist=%s", getaction(__entry->reason),
  9426. + __entry->starthist ? "start" : "stop")
  9427. +);
  9428. +#endif
  9429. +
  9430. +#ifndef CONFIG_MISSED_TIMER_OFFSETS_HIST
  9431. +#define trace_hrtimer_interrupt(a, b, c, d)
  9432. +#else
  9433. +TRACE_EVENT(hrtimer_interrupt,
  9434. +
  9435. + TP_PROTO(int cpu, long long offset, struct task_struct *curr,
  9436. + struct task_struct *task),
  9437. +
  9438. + TP_ARGS(cpu, offset, curr, task),
  9439. +
  9440. + TP_STRUCT__entry(
  9441. + __field(int, cpu)
  9442. + __field(long long, offset)
  9443. + __array(char, ccomm, TASK_COMM_LEN)
  9444. + __field(int, cprio)
  9445. + __array(char, tcomm, TASK_COMM_LEN)
  9446. + __field(int, tprio)
  9447. + ),
  9448. +
  9449. + TP_fast_assign(
  9450. + __entry->cpu = cpu;
  9451. + __entry->offset = offset;
  9452. + memcpy(__entry->ccomm, curr->comm, TASK_COMM_LEN);
  9453. + __entry->cprio = curr->prio;
  9454. + memcpy(__entry->tcomm, task != NULL ? task->comm : "<none>",
  9455. + task != NULL ? TASK_COMM_LEN : 7);
  9456. + __entry->tprio = task != NULL ? task->prio : -1;
  9457. + ),
  9458. +
  9459. + TP_printk("cpu=%d offset=%lld curr=%s[%d] thread=%s[%d]",
  9460. + __entry->cpu, __entry->offset, __entry->ccomm,
  9461. + __entry->cprio, __entry->tcomm, __entry->tprio)
  9462. +);
  9463. +#endif
  9464. +
  9465. +#endif /* _TRACE_HIST_H */
  9466. +
  9467. +/* This part must be outside protection */
  9468. +#include <trace/define_trace.h>
  9469. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/trace/events/latency_hist.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/trace/events/latency_hist.h
  9470. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/include/trace/events/latency_hist.h 1970-01-01 01:00:00.000000000 +0100
  9471. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/include/trace/events/latency_hist.h 2017-04-18 17:54:26.000000000 +0200
  9472. @@ -0,0 +1,29 @@
  9473. +#ifndef _LATENCY_HIST_H
  9474. +#define _LATENCY_HIST_H
  9475. +
  9476. +enum hist_action {
  9477. + IRQS_ON,
  9478. + PREEMPT_ON,
  9479. + TRACE_STOP,
  9480. + IRQS_OFF,
  9481. + PREEMPT_OFF,
  9482. + TRACE_START,
  9483. +};
  9484. +
  9485. +static char *actions[] = {
  9486. + "IRQS_ON",
  9487. + "PREEMPT_ON",
  9488. + "TRACE_STOP",
  9489. + "IRQS_OFF",
  9490. + "PREEMPT_OFF",
  9491. + "TRACE_START",
  9492. +};
  9493. +
  9494. +static inline char *getaction(int action)
  9495. +{
  9496. + if (action >= 0 && action <= sizeof(actions)/sizeof(actions[0]))
  9497. + return actions[action];
  9498. + return "unknown";
  9499. +}
  9500. +
  9501. +#endif /* _LATENCY_HIST_H */
  9502. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/init/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/init/Kconfig
  9503. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/init/Kconfig 2017-04-16 10:38:29.000000000 +0200
  9504. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/init/Kconfig 2017-04-18 17:54:26.000000000 +0200
  9505. @@ -506,7 +506,7 @@
  9506. config RCU_EXPERT
  9507. bool "Make expert-level adjustments to RCU configuration"
  9508. - default n
  9509. + default y if PREEMPT_RT_FULL
  9510. help
  9511. This option needs to be enabled if you wish to make
  9512. expert-level adjustments to RCU configuration. By default,
  9513. @@ -623,7 +623,7 @@
  9514. config RCU_FAST_NO_HZ
  9515. bool "Accelerate last non-dyntick-idle CPU's grace periods"
  9516. - depends on NO_HZ_COMMON && SMP && RCU_EXPERT
  9517. + depends on NO_HZ_COMMON && SMP && RCU_EXPERT && !PREEMPT_RT_FULL
  9518. default n
  9519. help
  9520. This option permits CPUs to enter dynticks-idle state even if
  9521. @@ -650,7 +650,7 @@
  9522. config RCU_BOOST
  9523. bool "Enable RCU priority boosting"
  9524. depends on RT_MUTEXES && PREEMPT_RCU && RCU_EXPERT
  9525. - default n
  9526. + default y if PREEMPT_RT_FULL
  9527. help
  9528. This option boosts the priority of preempted RCU readers that
  9529. block the current preemptible RCU grace period for too long.
  9530. @@ -781,19 +781,6 @@
  9531. endchoice
  9532. -config RCU_EXPEDITE_BOOT
  9533. - bool
  9534. - default n
  9535. - help
  9536. - This option enables expedited grace periods at boot time,
  9537. - as if rcu_expedite_gp() had been invoked early in boot.
  9538. - The corresponding rcu_unexpedite_gp() is invoked from
  9539. - rcu_end_inkernel_boot(), which is intended to be invoked
  9540. - at the end of the kernel-only boot sequence, just before
  9541. - init is exec'ed.
  9542. -
  9543. - Accept the default if unsure.
  9544. -
  9545. endmenu # "RCU Subsystem"
  9546. config BUILD_BIN2C
  9547. @@ -1064,6 +1051,7 @@
  9548. config RT_GROUP_SCHED
  9549. bool "Group scheduling for SCHED_RR/FIFO"
  9550. depends on CGROUP_SCHED
  9551. + depends on !PREEMPT_RT_FULL
  9552. default n
  9553. help
  9554. This feature lets you explicitly allocate real CPU bandwidth
  9555. @@ -1772,6 +1760,7 @@
  9556. config SLAB
  9557. bool "SLAB"
  9558. + depends on !PREEMPT_RT_FULL
  9559. select HAVE_HARDENED_USERCOPY_ALLOCATOR
  9560. help
  9561. The regular slab allocator that is established and known to work
  9562. @@ -1792,6 +1781,7 @@
  9563. config SLOB
  9564. depends on EXPERT
  9565. bool "SLOB (Simple Allocator)"
  9566. + depends on !PREEMPT_RT_FULL
  9567. help
  9568. SLOB replaces the stock allocator with a drastically simpler
  9569. allocator. SLOB is generally more space efficient but
  9570. @@ -1810,7 +1800,7 @@
  9571. config SLUB_CPU_PARTIAL
  9572. default y
  9573. - depends on SLUB && SMP
  9574. + depends on SLUB && SMP && !PREEMPT_RT_FULL
  9575. bool "SLUB per cpu partial cache"
  9576. help
  9577. Per cpu partial caches accellerate objects allocation and freeing
  9578. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/init/Makefile linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/init/Makefile
  9579. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/init/Makefile 2017-04-16 10:38:29.000000000 +0200
  9580. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/init/Makefile 2017-04-18 17:54:26.000000000 +0200
  9581. @@ -35,4 +35,4 @@
  9582. include/generated/compile.h: FORCE
  9583. @$($(quiet)chk_compile.h)
  9584. $(Q)$(CONFIG_SHELL) $(srctree)/scripts/mkcompile_h $@ \
  9585. - "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CC) $(KBUILD_CFLAGS)"
  9586. + "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CONFIG_PREEMPT_RT_FULL)" "$(CC) $(KBUILD_CFLAGS)"
  9587. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/init/main.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/init/main.c
  9588. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/init/main.c 2017-04-16 10:38:29.000000000 +0200
  9589. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/init/main.c 2017-04-18 17:54:26.000000000 +0200
  9590. @@ -507,6 +507,7 @@
  9591. setup_command_line(command_line);
  9592. setup_nr_cpu_ids();
  9593. setup_per_cpu_areas();
  9594. + softirq_early_init();
  9595. boot_cpu_state_init();
  9596. smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */
  9597. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/ipc/sem.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/ipc/sem.c
  9598. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/ipc/sem.c 2017-04-16 10:38:29.000000000 +0200
  9599. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/ipc/sem.c 2017-04-18 17:54:26.000000000 +0200
  9600. @@ -712,6 +712,13 @@
  9601. static void wake_up_sem_queue_prepare(struct list_head *pt,
  9602. struct sem_queue *q, int error)
  9603. {
  9604. +#ifdef CONFIG_PREEMPT_RT_BASE
  9605. + struct task_struct *p = q->sleeper;
  9606. + get_task_struct(p);
  9607. + q->status = error;
  9608. + wake_up_process(p);
  9609. + put_task_struct(p);
  9610. +#else
  9611. if (list_empty(pt)) {
  9612. /*
  9613. * Hold preempt off so that we don't get preempted and have the
  9614. @@ -723,6 +730,7 @@
  9615. q->pid = error;
  9616. list_add_tail(&q->list, pt);
  9617. +#endif
  9618. }
  9619. /**
  9620. @@ -736,6 +744,7 @@
  9621. */
  9622. static void wake_up_sem_queue_do(struct list_head *pt)
  9623. {
  9624. +#ifndef CONFIG_PREEMPT_RT_BASE
  9625. struct sem_queue *q, *t;
  9626. int did_something;
  9627. @@ -748,6 +757,7 @@
  9628. }
  9629. if (did_something)
  9630. preempt_enable();
  9631. +#endif
  9632. }
  9633. static void unlink_queue(struct sem_array *sma, struct sem_queue *q)
  9634. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/Kconfig.locks linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/Kconfig.locks
  9635. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/Kconfig.locks 2017-04-16 10:38:29.000000000 +0200
  9636. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/Kconfig.locks 2017-04-18 17:54:26.000000000 +0200
  9637. @@ -225,11 +225,11 @@
  9638. config MUTEX_SPIN_ON_OWNER
  9639. def_bool y
  9640. - depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW
  9641. + depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW && !PREEMPT_RT_FULL
  9642. config RWSEM_SPIN_ON_OWNER
  9643. def_bool y
  9644. - depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW
  9645. + depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW && !PREEMPT_RT_FULL
  9646. config LOCK_SPIN_ON_OWNER
  9647. def_bool y
  9648. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/Kconfig.preempt linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/Kconfig.preempt
  9649. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/Kconfig.preempt 2017-04-16 10:38:29.000000000 +0200
  9650. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/Kconfig.preempt 2017-04-18 17:54:26.000000000 +0200
  9651. @@ -1,3 +1,16 @@
  9652. +config PREEMPT
  9653. + bool
  9654. + select PREEMPT_COUNT
  9655. +
  9656. +config PREEMPT_RT_BASE
  9657. + bool
  9658. + select PREEMPT
  9659. +
  9660. +config HAVE_PREEMPT_LAZY
  9661. + bool
  9662. +
  9663. +config PREEMPT_LAZY
  9664. + def_bool y if HAVE_PREEMPT_LAZY && PREEMPT_RT_FULL
  9665. choice
  9666. prompt "Preemption Model"
  9667. @@ -33,9 +46,9 @@
  9668. Select this if you are building a kernel for a desktop system.
  9669. -config PREEMPT
  9670. +config PREEMPT__LL
  9671. bool "Preemptible Kernel (Low-Latency Desktop)"
  9672. - select PREEMPT_COUNT
  9673. + select PREEMPT
  9674. select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK
  9675. help
  9676. This option reduces the latency of the kernel by making
  9677. @@ -52,6 +65,22 @@
  9678. embedded system with latency requirements in the milliseconds
  9679. range.
  9680. +config PREEMPT_RTB
  9681. + bool "Preemptible Kernel (Basic RT)"
  9682. + select PREEMPT_RT_BASE
  9683. + help
  9684. + This option is basically the same as (Low-Latency Desktop) but
  9685. + enables changes which are preliminary for the full preemptible
  9686. + RT kernel.
  9687. +
  9688. +config PREEMPT_RT_FULL
  9689. + bool "Fully Preemptible Kernel (RT)"
  9690. + depends on IRQ_FORCED_THREADING
  9691. + select PREEMPT_RT_BASE
  9692. + select PREEMPT_RCU
  9693. + help
  9694. + All and everything
  9695. +
  9696. endchoice
  9697. config PREEMPT_COUNT
  9698. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/cgroup.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/cgroup.c
  9699. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/cgroup.c 2017-04-16 10:38:29.000000000 +0200
  9700. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/cgroup.c 2017-04-18 17:54:26.000000000 +0200
  9701. @@ -5040,10 +5040,10 @@
  9702. queue_work(cgroup_destroy_wq, &css->destroy_work);
  9703. }
  9704. -static void css_release_work_fn(struct work_struct *work)
  9705. +static void css_release_work_fn(struct swork_event *sev)
  9706. {
  9707. struct cgroup_subsys_state *css =
  9708. - container_of(work, struct cgroup_subsys_state, destroy_work);
  9709. + container_of(sev, struct cgroup_subsys_state, destroy_swork);
  9710. struct cgroup_subsys *ss = css->ss;
  9711. struct cgroup *cgrp = css->cgroup;
  9712. @@ -5086,8 +5086,8 @@
  9713. struct cgroup_subsys_state *css =
  9714. container_of(ref, struct cgroup_subsys_state, refcnt);
  9715. - INIT_WORK(&css->destroy_work, css_release_work_fn);
  9716. - queue_work(cgroup_destroy_wq, &css->destroy_work);
  9717. + INIT_SWORK(&css->destroy_swork, css_release_work_fn);
  9718. + swork_queue(&css->destroy_swork);
  9719. }
  9720. static void init_and_link_css(struct cgroup_subsys_state *css,
  9721. @@ -5739,6 +5739,7 @@
  9722. */
  9723. cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
  9724. BUG_ON(!cgroup_destroy_wq);
  9725. + BUG_ON(swork_get());
  9726. /*
  9727. * Used to destroy pidlists and separate to serve as flush domain.
  9728. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/cpu.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/cpu.c
  9729. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/cpu.c 2017-04-16 10:38:29.000000000 +0200
  9730. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/cpu.c 2017-04-18 17:54:26.000000000 +0200
  9731. @@ -239,6 +239,289 @@
  9732. #define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map)
  9733. #define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map)
  9734. +/**
  9735. + * hotplug_pcp - per cpu hotplug descriptor
  9736. + * @unplug: set when pin_current_cpu() needs to sync tasks
  9737. + * @sync_tsk: the task that waits for tasks to finish pinned sections
  9738. + * @refcount: counter of tasks in pinned sections
  9739. + * @grab_lock: set when the tasks entering pinned sections should wait
  9740. + * @synced: notifier for @sync_tsk to tell cpu_down it's finished
  9741. + * @mutex: the mutex to make tasks wait (used when @grab_lock is true)
  9742. + * @mutex_init: zero if the mutex hasn't been initialized yet.
  9743. + *
  9744. + * Although @unplug and @sync_tsk may point to the same task, the @unplug
  9745. + * is used as a flag and still exists after @sync_tsk has exited and
  9746. + * @sync_tsk set to NULL.
  9747. + */
  9748. +struct hotplug_pcp {
  9749. + struct task_struct *unplug;
  9750. + struct task_struct *sync_tsk;
  9751. + int refcount;
  9752. + int grab_lock;
  9753. + struct completion synced;
  9754. + struct completion unplug_wait;
  9755. +#ifdef CONFIG_PREEMPT_RT_FULL
  9756. + /*
  9757. + * Note, on PREEMPT_RT, the hotplug lock must save the state of
  9758. + * the task, otherwise the mutex will cause the task to fail
  9759. + * to sleep when required. (Because it's called from migrate_disable())
  9760. + *
  9761. + * The spinlock_t on PREEMPT_RT is a mutex that saves the task's
  9762. + * state.
  9763. + */
  9764. + spinlock_t lock;
  9765. +#else
  9766. + struct mutex mutex;
  9767. +#endif
  9768. + int mutex_init;
  9769. +};
  9770. +
  9771. +#ifdef CONFIG_PREEMPT_RT_FULL
  9772. +# define hotplug_lock(hp) rt_spin_lock__no_mg(&(hp)->lock)
  9773. +# define hotplug_unlock(hp) rt_spin_unlock__no_mg(&(hp)->lock)
  9774. +#else
  9775. +# define hotplug_lock(hp) mutex_lock(&(hp)->mutex)
  9776. +# define hotplug_unlock(hp) mutex_unlock(&(hp)->mutex)
  9777. +#endif
  9778. +
  9779. +static DEFINE_PER_CPU(struct hotplug_pcp, hotplug_pcp);
  9780. +
  9781. +/**
  9782. + * pin_current_cpu - Prevent the current cpu from being unplugged
  9783. + *
  9784. + * Lightweight version of get_online_cpus() to prevent cpu from being
  9785. + * unplugged when code runs in a migration disabled region.
  9786. + *
  9787. + * Must be called with preemption disabled (preempt_count = 1)!
  9788. + */
  9789. +void pin_current_cpu(void)
  9790. +{
  9791. + struct hotplug_pcp *hp;
  9792. + int force = 0;
  9793. +
  9794. +retry:
  9795. + hp = this_cpu_ptr(&hotplug_pcp);
  9796. +
  9797. + if (!hp->unplug || hp->refcount || force || preempt_count() > 1 ||
  9798. + hp->unplug == current) {
  9799. + hp->refcount++;
  9800. + return;
  9801. + }
  9802. + if (hp->grab_lock) {
  9803. + preempt_enable();
  9804. + hotplug_lock(hp);
  9805. + hotplug_unlock(hp);
  9806. + } else {
  9807. + preempt_enable();
  9808. + /*
  9809. + * Try to push this task off of this CPU.
  9810. + */
  9811. + if (!migrate_me()) {
  9812. + preempt_disable();
  9813. + hp = this_cpu_ptr(&hotplug_pcp);
  9814. + if (!hp->grab_lock) {
  9815. + /*
  9816. + * Just let it continue it's already pinned
  9817. + * or about to sleep.
  9818. + */
  9819. + force = 1;
  9820. + goto retry;
  9821. + }
  9822. + preempt_enable();
  9823. + }
  9824. + }
  9825. + preempt_disable();
  9826. + goto retry;
  9827. +}
  9828. +
  9829. +/**
  9830. + * unpin_current_cpu - Allow unplug of current cpu
  9831. + *
  9832. + * Must be called with preemption or interrupts disabled!
  9833. + */
  9834. +void unpin_current_cpu(void)
  9835. +{
  9836. + struct hotplug_pcp *hp = this_cpu_ptr(&hotplug_pcp);
  9837. +
  9838. + WARN_ON(hp->refcount <= 0);
  9839. +
  9840. + /* This is safe. sync_unplug_thread is pinned to this cpu */
  9841. + if (!--hp->refcount && hp->unplug && hp->unplug != current)
  9842. + wake_up_process(hp->unplug);
  9843. +}
  9844. +
  9845. +static void wait_for_pinned_cpus(struct hotplug_pcp *hp)
  9846. +{
  9847. + set_current_state(TASK_UNINTERRUPTIBLE);
  9848. + while (hp->refcount) {
  9849. + schedule_preempt_disabled();
  9850. + set_current_state(TASK_UNINTERRUPTIBLE);
  9851. + }
  9852. +}
  9853. +
  9854. +static int sync_unplug_thread(void *data)
  9855. +{
  9856. + struct hotplug_pcp *hp = data;
  9857. +
  9858. + wait_for_completion(&hp->unplug_wait);
  9859. + preempt_disable();
  9860. + hp->unplug = current;
  9861. + wait_for_pinned_cpus(hp);
  9862. +
  9863. + /*
  9864. + * This thread will synchronize the cpu_down() with threads
  9865. + * that have pinned the CPU. When the pinned CPU count reaches
  9866. + * zero, we inform the cpu_down code to continue to the next step.
  9867. + */
  9868. + set_current_state(TASK_UNINTERRUPTIBLE);
  9869. + preempt_enable();
  9870. + complete(&hp->synced);
  9871. +
  9872. + /*
  9873. + * If all succeeds, the next step will need tasks to wait till
  9874. + * the CPU is offline before continuing. To do this, the grab_lock
  9875. + * is set and tasks going into pin_current_cpu() will block on the
  9876. + * mutex. But we still need to wait for those that are already in
  9877. + * pinned CPU sections. If the cpu_down() failed, the kthread_should_stop()
  9878. + * will kick this thread out.
  9879. + */
  9880. + while (!hp->grab_lock && !kthread_should_stop()) {
  9881. + schedule();
  9882. + set_current_state(TASK_UNINTERRUPTIBLE);
  9883. + }
  9884. +
  9885. + /* Make sure grab_lock is seen before we see a stale completion */
  9886. + smp_mb();
  9887. +
  9888. + /*
  9889. + * Now just before cpu_down() enters stop machine, we need to make
  9890. + * sure all tasks that are in pinned CPU sections are out, and new
  9891. + * tasks will now grab the lock, keeping them from entering pinned
  9892. + * CPU sections.
  9893. + */
  9894. + if (!kthread_should_stop()) {
  9895. + preempt_disable();
  9896. + wait_for_pinned_cpus(hp);
  9897. + preempt_enable();
  9898. + complete(&hp->synced);
  9899. + }
  9900. +
  9901. + set_current_state(TASK_UNINTERRUPTIBLE);
  9902. + while (!kthread_should_stop()) {
  9903. + schedule();
  9904. + set_current_state(TASK_UNINTERRUPTIBLE);
  9905. + }
  9906. + set_current_state(TASK_RUNNING);
  9907. +
  9908. + /*
  9909. + * Force this thread off this CPU as it's going down and
  9910. + * we don't want any more work on this CPU.
  9911. + */
  9912. + current->flags &= ~PF_NO_SETAFFINITY;
  9913. + set_cpus_allowed_ptr(current, cpu_present_mask);
  9914. + migrate_me();
  9915. + return 0;
  9916. +}
  9917. +
  9918. +static void __cpu_unplug_sync(struct hotplug_pcp *hp)
  9919. +{
  9920. + wake_up_process(hp->sync_tsk);
  9921. + wait_for_completion(&hp->synced);
  9922. +}
  9923. +
  9924. +static void __cpu_unplug_wait(unsigned int cpu)
  9925. +{
  9926. + struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu);
  9927. +
  9928. + complete(&hp->unplug_wait);
  9929. + wait_for_completion(&hp->synced);
  9930. +}
  9931. +
  9932. +/*
  9933. + * Start the sync_unplug_thread on the target cpu and wait for it to
  9934. + * complete.
  9935. + */
  9936. +static int cpu_unplug_begin(unsigned int cpu)
  9937. +{
  9938. + struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu);
  9939. + int err;
  9940. +
  9941. + /* Protected by cpu_hotplug.lock */
  9942. + if (!hp->mutex_init) {
  9943. +#ifdef CONFIG_PREEMPT_RT_FULL
  9944. + spin_lock_init(&hp->lock);
  9945. +#else
  9946. + mutex_init(&hp->mutex);
  9947. +#endif
  9948. + hp->mutex_init = 1;
  9949. + }
  9950. +
  9951. + /* Inform the scheduler to migrate tasks off this CPU */
  9952. + tell_sched_cpu_down_begin(cpu);
  9953. +
  9954. + init_completion(&hp->synced);
  9955. + init_completion(&hp->unplug_wait);
  9956. +
  9957. + hp->sync_tsk = kthread_create(sync_unplug_thread, hp, "sync_unplug/%d", cpu);
  9958. + if (IS_ERR(hp->sync_tsk)) {
  9959. + err = PTR_ERR(hp->sync_tsk);
  9960. + hp->sync_tsk = NULL;
  9961. + return err;
  9962. + }
  9963. + kthread_bind(hp->sync_tsk, cpu);
  9964. +
  9965. + /*
  9966. + * Wait for tasks to get out of the pinned sections,
  9967. + * it's still OK if new tasks enter. Some CPU notifiers will
  9968. + * wait for tasks that are going to enter these sections and
  9969. + * we must not have them block.
  9970. + */
  9971. + wake_up_process(hp->sync_tsk);
  9972. + return 0;
  9973. +}
  9974. +
  9975. +static void cpu_unplug_sync(unsigned int cpu)
  9976. +{
  9977. + struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu);
  9978. +
  9979. + init_completion(&hp->synced);
  9980. + /* The completion needs to be initialzied before setting grab_lock */
  9981. + smp_wmb();
  9982. +
  9983. + /* Grab the mutex before setting grab_lock */
  9984. + hotplug_lock(hp);
  9985. + hp->grab_lock = 1;
  9986. +
  9987. + /*
  9988. + * The CPU notifiers have been completed.
  9989. + * Wait for tasks to get out of pinned CPU sections and have new
  9990. + * tasks block until the CPU is completely down.
  9991. + */
  9992. + __cpu_unplug_sync(hp);
  9993. +
  9994. + /* All done with the sync thread */
  9995. + kthread_stop(hp->sync_tsk);
  9996. + hp->sync_tsk = NULL;
  9997. +}
  9998. +
  9999. +static void cpu_unplug_done(unsigned int cpu)
  10000. +{
  10001. + struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu);
  10002. +
  10003. + hp->unplug = NULL;
  10004. + /* Let all tasks know cpu unplug is finished before cleaning up */
  10005. + smp_wmb();
  10006. +
  10007. + if (hp->sync_tsk)
  10008. + kthread_stop(hp->sync_tsk);
  10009. +
  10010. + if (hp->grab_lock) {
  10011. + hotplug_unlock(hp);
  10012. + /* protected by cpu_hotplug.lock */
  10013. + hp->grab_lock = 0;
  10014. + }
  10015. + tell_sched_cpu_down_done(cpu);
  10016. +}
  10017. void get_online_cpus(void)
  10018. {
  10019. @@ -789,10 +1072,14 @@
  10020. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  10021. int err;
  10022. + __cpu_unplug_wait(cpu);
  10023. /* Park the smpboot threads */
  10024. kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
  10025. smpboot_park_threads(cpu);
  10026. + /* Notifiers are done. Don't let any more tasks pin this CPU. */
  10027. + cpu_unplug_sync(cpu);
  10028. +
  10029. /*
  10030. * Prevent irq alloc/free while the dying cpu reorganizes the
  10031. * interrupt affinities.
  10032. @@ -877,6 +1164,9 @@
  10033. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  10034. int prev_state, ret = 0;
  10035. bool hasdied = false;
  10036. + int mycpu;
  10037. + cpumask_var_t cpumask;
  10038. + cpumask_var_t cpumask_org;
  10039. if (num_online_cpus() == 1)
  10040. return -EBUSY;
  10041. @@ -884,7 +1174,34 @@
  10042. if (!cpu_present(cpu))
  10043. return -EINVAL;
  10044. + /* Move the downtaker off the unplug cpu */
  10045. + if (!alloc_cpumask_var(&cpumask, GFP_KERNEL))
  10046. + return -ENOMEM;
  10047. + if (!alloc_cpumask_var(&cpumask_org, GFP_KERNEL)) {
  10048. + free_cpumask_var(cpumask);
  10049. + return -ENOMEM;
  10050. + }
  10051. +
  10052. + cpumask_copy(cpumask_org, tsk_cpus_allowed(current));
  10053. + cpumask_andnot(cpumask, cpu_online_mask, cpumask_of(cpu));
  10054. + set_cpus_allowed_ptr(current, cpumask);
  10055. + free_cpumask_var(cpumask);
  10056. + migrate_disable();
  10057. + mycpu = smp_processor_id();
  10058. + if (mycpu == cpu) {
  10059. + printk(KERN_ERR "Yuck! Still on unplug CPU\n!");
  10060. + migrate_enable();
  10061. + ret = -EBUSY;
  10062. + goto restore_cpus;
  10063. + }
  10064. +
  10065. + migrate_enable();
  10066. cpu_hotplug_begin();
  10067. + ret = cpu_unplug_begin(cpu);
  10068. + if (ret) {
  10069. + printk("cpu_unplug_begin(%d) failed\n", cpu);
  10070. + goto out_cancel;
  10071. + }
  10072. cpuhp_tasks_frozen = tasks_frozen;
  10073. @@ -923,10 +1240,15 @@
  10074. hasdied = prev_state != st->state && st->state == CPUHP_OFFLINE;
  10075. out:
  10076. + cpu_unplug_done(cpu);
  10077. +out_cancel:
  10078. cpu_hotplug_done();
  10079. /* This post dead nonsense must die */
  10080. if (!ret && hasdied)
  10081. cpu_notify_nofail(CPU_POST_DEAD, cpu);
  10082. +restore_cpus:
  10083. + set_cpus_allowed_ptr(current, cpumask_org);
  10084. + free_cpumask_var(cpumask_org);
  10085. return ret;
  10086. }
  10087. @@ -1240,6 +1562,8 @@
  10088. #endif /* CONFIG_PM_SLEEP_SMP */
  10089. +int __boot_cpu_id;
  10090. +
  10091. #endif /* CONFIG_SMP */
  10092. /* Boot processor state steps */
  10093. @@ -1923,6 +2247,10 @@
  10094. set_cpu_active(cpu, true);
  10095. set_cpu_present(cpu, true);
  10096. set_cpu_possible(cpu, true);
  10097. +
  10098. +#ifdef CONFIG_SMP
  10099. + __boot_cpu_id = cpu;
  10100. +#endif
  10101. }
  10102. /*
  10103. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/cpuset.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/cpuset.c
  10104. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/cpuset.c 2017-04-16 10:38:29.000000000 +0200
  10105. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/cpuset.c 2017-04-18 17:54:26.000000000 +0200
  10106. @@ -284,7 +284,7 @@
  10107. */
  10108. static DEFINE_MUTEX(cpuset_mutex);
  10109. -static DEFINE_SPINLOCK(callback_lock);
  10110. +static DEFINE_RAW_SPINLOCK(callback_lock);
  10111. static struct workqueue_struct *cpuset_migrate_mm_wq;
  10112. @@ -907,9 +907,9 @@
  10113. continue;
  10114. rcu_read_unlock();
  10115. - spin_lock_irq(&callback_lock);
  10116. + raw_spin_lock_irq(&callback_lock);
  10117. cpumask_copy(cp->effective_cpus, new_cpus);
  10118. - spin_unlock_irq(&callback_lock);
  10119. + raw_spin_unlock_irq(&callback_lock);
  10120. WARN_ON(!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) &&
  10121. !cpumask_equal(cp->cpus_allowed, cp->effective_cpus));
  10122. @@ -974,9 +974,9 @@
  10123. if (retval < 0)
  10124. return retval;
  10125. - spin_lock_irq(&callback_lock);
  10126. + raw_spin_lock_irq(&callback_lock);
  10127. cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);
  10128. - spin_unlock_irq(&callback_lock);
  10129. + raw_spin_unlock_irq(&callback_lock);
  10130. /* use trialcs->cpus_allowed as a temp variable */
  10131. update_cpumasks_hier(cs, trialcs->cpus_allowed);
  10132. @@ -1176,9 +1176,9 @@
  10133. continue;
  10134. rcu_read_unlock();
  10135. - spin_lock_irq(&callback_lock);
  10136. + raw_spin_lock_irq(&callback_lock);
  10137. cp->effective_mems = *new_mems;
  10138. - spin_unlock_irq(&callback_lock);
  10139. + raw_spin_unlock_irq(&callback_lock);
  10140. WARN_ON(!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) &&
  10141. !nodes_equal(cp->mems_allowed, cp->effective_mems));
  10142. @@ -1246,9 +1246,9 @@
  10143. if (retval < 0)
  10144. goto done;
  10145. - spin_lock_irq(&callback_lock);
  10146. + raw_spin_lock_irq(&callback_lock);
  10147. cs->mems_allowed = trialcs->mems_allowed;
  10148. - spin_unlock_irq(&callback_lock);
  10149. + raw_spin_unlock_irq(&callback_lock);
  10150. /* use trialcs->mems_allowed as a temp variable */
  10151. update_nodemasks_hier(cs, &trialcs->mems_allowed);
  10152. @@ -1339,9 +1339,9 @@
  10153. spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs))
  10154. || (is_spread_page(cs) != is_spread_page(trialcs)));
  10155. - spin_lock_irq(&callback_lock);
  10156. + raw_spin_lock_irq(&callback_lock);
  10157. cs->flags = trialcs->flags;
  10158. - spin_unlock_irq(&callback_lock);
  10159. + raw_spin_unlock_irq(&callback_lock);
  10160. if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
  10161. rebuild_sched_domains_locked();
  10162. @@ -1756,7 +1756,7 @@
  10163. cpuset_filetype_t type = seq_cft(sf)->private;
  10164. int ret = 0;
  10165. - spin_lock_irq(&callback_lock);
  10166. + raw_spin_lock_irq(&callback_lock);
  10167. switch (type) {
  10168. case FILE_CPULIST:
  10169. @@ -1775,7 +1775,7 @@
  10170. ret = -EINVAL;
  10171. }
  10172. - spin_unlock_irq(&callback_lock);
  10173. + raw_spin_unlock_irq(&callback_lock);
  10174. return ret;
  10175. }
  10176. @@ -1989,12 +1989,12 @@
  10177. cpuset_inc();
  10178. - spin_lock_irq(&callback_lock);
  10179. + raw_spin_lock_irq(&callback_lock);
  10180. if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) {
  10181. cpumask_copy(cs->effective_cpus, parent->effective_cpus);
  10182. cs->effective_mems = parent->effective_mems;
  10183. }
  10184. - spin_unlock_irq(&callback_lock);
  10185. + raw_spin_unlock_irq(&callback_lock);
  10186. if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
  10187. goto out_unlock;
  10188. @@ -2021,12 +2021,12 @@
  10189. }
  10190. rcu_read_unlock();
  10191. - spin_lock_irq(&callback_lock);
  10192. + raw_spin_lock_irq(&callback_lock);
  10193. cs->mems_allowed = parent->mems_allowed;
  10194. cs->effective_mems = parent->mems_allowed;
  10195. cpumask_copy(cs->cpus_allowed, parent->cpus_allowed);
  10196. cpumask_copy(cs->effective_cpus, parent->cpus_allowed);
  10197. - spin_unlock_irq(&callback_lock);
  10198. + raw_spin_unlock_irq(&callback_lock);
  10199. out_unlock:
  10200. mutex_unlock(&cpuset_mutex);
  10201. return 0;
  10202. @@ -2065,7 +2065,7 @@
  10203. static void cpuset_bind(struct cgroup_subsys_state *root_css)
  10204. {
  10205. mutex_lock(&cpuset_mutex);
  10206. - spin_lock_irq(&callback_lock);
  10207. + raw_spin_lock_irq(&callback_lock);
  10208. if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) {
  10209. cpumask_copy(top_cpuset.cpus_allowed, cpu_possible_mask);
  10210. @@ -2076,7 +2076,7 @@
  10211. top_cpuset.mems_allowed = top_cpuset.effective_mems;
  10212. }
  10213. - spin_unlock_irq(&callback_lock);
  10214. + raw_spin_unlock_irq(&callback_lock);
  10215. mutex_unlock(&cpuset_mutex);
  10216. }
  10217. @@ -2177,12 +2177,12 @@
  10218. {
  10219. bool is_empty;
  10220. - spin_lock_irq(&callback_lock);
  10221. + raw_spin_lock_irq(&callback_lock);
  10222. cpumask_copy(cs->cpus_allowed, new_cpus);
  10223. cpumask_copy(cs->effective_cpus, new_cpus);
  10224. cs->mems_allowed = *new_mems;
  10225. cs->effective_mems = *new_mems;
  10226. - spin_unlock_irq(&callback_lock);
  10227. + raw_spin_unlock_irq(&callback_lock);
  10228. /*
  10229. * Don't call update_tasks_cpumask() if the cpuset becomes empty,
  10230. @@ -2219,10 +2219,10 @@
  10231. if (nodes_empty(*new_mems))
  10232. *new_mems = parent_cs(cs)->effective_mems;
  10233. - spin_lock_irq(&callback_lock);
  10234. + raw_spin_lock_irq(&callback_lock);
  10235. cpumask_copy(cs->effective_cpus, new_cpus);
  10236. cs->effective_mems = *new_mems;
  10237. - spin_unlock_irq(&callback_lock);
  10238. + raw_spin_unlock_irq(&callback_lock);
  10239. if (cpus_updated)
  10240. update_tasks_cpumask(cs);
  10241. @@ -2308,21 +2308,21 @@
  10242. /* synchronize cpus_allowed to cpu_active_mask */
  10243. if (cpus_updated) {
  10244. - spin_lock_irq(&callback_lock);
  10245. + raw_spin_lock_irq(&callback_lock);
  10246. if (!on_dfl)
  10247. cpumask_copy(top_cpuset.cpus_allowed, &new_cpus);
  10248. cpumask_copy(top_cpuset.effective_cpus, &new_cpus);
  10249. - spin_unlock_irq(&callback_lock);
  10250. + raw_spin_unlock_irq(&callback_lock);
  10251. /* we don't mess with cpumasks of tasks in top_cpuset */
  10252. }
  10253. /* synchronize mems_allowed to N_MEMORY */
  10254. if (mems_updated) {
  10255. - spin_lock_irq(&callback_lock);
  10256. + raw_spin_lock_irq(&callback_lock);
  10257. if (!on_dfl)
  10258. top_cpuset.mems_allowed = new_mems;
  10259. top_cpuset.effective_mems = new_mems;
  10260. - spin_unlock_irq(&callback_lock);
  10261. + raw_spin_unlock_irq(&callback_lock);
  10262. update_tasks_nodemask(&top_cpuset);
  10263. }
  10264. @@ -2420,11 +2420,11 @@
  10265. {
  10266. unsigned long flags;
  10267. - spin_lock_irqsave(&callback_lock, flags);
  10268. + raw_spin_lock_irqsave(&callback_lock, flags);
  10269. rcu_read_lock();
  10270. guarantee_online_cpus(task_cs(tsk), pmask);
  10271. rcu_read_unlock();
  10272. - spin_unlock_irqrestore(&callback_lock, flags);
  10273. + raw_spin_unlock_irqrestore(&callback_lock, flags);
  10274. }
  10275. void cpuset_cpus_allowed_fallback(struct task_struct *tsk)
  10276. @@ -2472,11 +2472,11 @@
  10277. nodemask_t mask;
  10278. unsigned long flags;
  10279. - spin_lock_irqsave(&callback_lock, flags);
  10280. + raw_spin_lock_irqsave(&callback_lock, flags);
  10281. rcu_read_lock();
  10282. guarantee_online_mems(task_cs(tsk), &mask);
  10283. rcu_read_unlock();
  10284. - spin_unlock_irqrestore(&callback_lock, flags);
  10285. + raw_spin_unlock_irqrestore(&callback_lock, flags);
  10286. return mask;
  10287. }
  10288. @@ -2568,14 +2568,14 @@
  10289. return true;
  10290. /* Not hardwall and node outside mems_allowed: scan up cpusets */
  10291. - spin_lock_irqsave(&callback_lock, flags);
  10292. + raw_spin_lock_irqsave(&callback_lock, flags);
  10293. rcu_read_lock();
  10294. cs = nearest_hardwall_ancestor(task_cs(current));
  10295. allowed = node_isset(node, cs->mems_allowed);
  10296. rcu_read_unlock();
  10297. - spin_unlock_irqrestore(&callback_lock, flags);
  10298. + raw_spin_unlock_irqrestore(&callback_lock, flags);
  10299. return allowed;
  10300. }
  10301. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/debug/kdb/kdb_io.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/debug/kdb/kdb_io.c
  10302. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/debug/kdb/kdb_io.c 2017-04-16 10:38:29.000000000 +0200
  10303. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/debug/kdb/kdb_io.c 2017-04-18 17:54:26.000000000 +0200
  10304. @@ -554,7 +554,6 @@
  10305. int linecount;
  10306. int colcount;
  10307. int logging, saved_loglevel = 0;
  10308. - int saved_trap_printk;
  10309. int got_printf_lock = 0;
  10310. int retlen = 0;
  10311. int fnd, len;
  10312. @@ -565,8 +564,6 @@
  10313. unsigned long uninitialized_var(flags);
  10314. preempt_disable();
  10315. - saved_trap_printk = kdb_trap_printk;
  10316. - kdb_trap_printk = 0;
  10317. /* Serialize kdb_printf if multiple cpus try to write at once.
  10318. * But if any cpu goes recursive in kdb, just print the output,
  10319. @@ -855,7 +852,6 @@
  10320. } else {
  10321. __release(kdb_printf_lock);
  10322. }
  10323. - kdb_trap_printk = saved_trap_printk;
  10324. preempt_enable();
  10325. return retlen;
  10326. }
  10327. @@ -865,9 +861,11 @@
  10328. va_list ap;
  10329. int r;
  10330. + kdb_trap_printk++;
  10331. va_start(ap, fmt);
  10332. r = vkdb_printf(KDB_MSGSRC_INTERNAL, fmt, ap);
  10333. va_end(ap);
  10334. + kdb_trap_printk--;
  10335. return r;
  10336. }
  10337. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/events/core.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/events/core.c
  10338. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/events/core.c 2017-04-16 10:38:29.000000000 +0200
  10339. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/events/core.c 2017-04-18 17:54:26.000000000 +0200
  10340. @@ -1050,6 +1050,7 @@
  10341. raw_spin_lock_init(&cpuctx->hrtimer_lock);
  10342. hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
  10343. timer->function = perf_mux_hrtimer_handler;
  10344. + timer->irqsafe = 1;
  10345. }
  10346. static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx)
  10347. @@ -8363,6 +8364,7 @@
  10348. hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  10349. hwc->hrtimer.function = perf_swevent_hrtimer;
  10350. + hwc->hrtimer.irqsafe = 1;
  10351. /*
  10352. * Since hrtimers have a fixed rate, we can do a static freq->period
  10353. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/exit.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/exit.c
  10354. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/exit.c 2017-04-16 10:38:29.000000000 +0200
  10355. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/exit.c 2017-04-18 17:54:26.000000000 +0200
  10356. @@ -143,7 +143,7 @@
  10357. * Do this under ->siglock, we can race with another thread
  10358. * doing sigqueue_free() if we have SIGQUEUE_PREALLOC signals.
  10359. */
  10360. - flush_sigqueue(&tsk->pending);
  10361. + flush_task_sigqueue(tsk);
  10362. tsk->sighand = NULL;
  10363. spin_unlock(&sighand->siglock);
  10364. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/fork.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/fork.c
  10365. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/fork.c 2017-04-16 10:38:29.000000000 +0200
  10366. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/fork.c 2017-04-18 17:54:26.000000000 +0200
  10367. @@ -76,6 +76,7 @@
  10368. #include <linux/compiler.h>
  10369. #include <linux/sysctl.h>
  10370. #include <linux/kcov.h>
  10371. +#include <linux/kprobes.h>
  10372. #include <asm/pgtable.h>
  10373. #include <asm/pgalloc.h>
  10374. @@ -376,13 +377,24 @@
  10375. if (atomic_dec_and_test(&sig->sigcnt))
  10376. free_signal_struct(sig);
  10377. }
  10378. -
  10379. +#ifdef CONFIG_PREEMPT_RT_BASE
  10380. +static
  10381. +#endif
  10382. void __put_task_struct(struct task_struct *tsk)
  10383. {
  10384. WARN_ON(!tsk->exit_state);
  10385. WARN_ON(atomic_read(&tsk->usage));
  10386. WARN_ON(tsk == current);
  10387. + /*
  10388. + * Remove function-return probe instances associated with this
  10389. + * task and put them back on the free list.
  10390. + */
  10391. + kprobe_flush_task(tsk);
  10392. +
  10393. + /* Task is done with its stack. */
  10394. + put_task_stack(tsk);
  10395. +
  10396. cgroup_free(tsk);
  10397. task_numa_free(tsk);
  10398. security_task_free(tsk);
  10399. @@ -393,7 +405,18 @@
  10400. if (!profile_handoff_task(tsk))
  10401. free_task(tsk);
  10402. }
  10403. +#ifndef CONFIG_PREEMPT_RT_BASE
  10404. EXPORT_SYMBOL_GPL(__put_task_struct);
  10405. +#else
  10406. +void __put_task_struct_cb(struct rcu_head *rhp)
  10407. +{
  10408. + struct task_struct *tsk = container_of(rhp, struct task_struct, put_rcu);
  10409. +
  10410. + __put_task_struct(tsk);
  10411. +
  10412. +}
  10413. +EXPORT_SYMBOL_GPL(__put_task_struct_cb);
  10414. +#endif
  10415. void __init __weak arch_task_cache_init(void) { }
  10416. @@ -852,6 +875,19 @@
  10417. }
  10418. EXPORT_SYMBOL_GPL(__mmdrop);
  10419. +#ifdef CONFIG_PREEMPT_RT_BASE
  10420. +/*
  10421. + * RCU callback for delayed mm drop. Not strictly rcu, but we don't
  10422. + * want another facility to make this work.
  10423. + */
  10424. +void __mmdrop_delayed(struct rcu_head *rhp)
  10425. +{
  10426. + struct mm_struct *mm = container_of(rhp, struct mm_struct, delayed_drop);
  10427. +
  10428. + __mmdrop(mm);
  10429. +}
  10430. +#endif
  10431. +
  10432. static inline void __mmput(struct mm_struct *mm)
  10433. {
  10434. VM_BUG_ON(atomic_read(&mm->mm_users));
  10435. @@ -1426,6 +1462,9 @@
  10436. */
  10437. static void posix_cpu_timers_init(struct task_struct *tsk)
  10438. {
  10439. +#ifdef CONFIG_PREEMPT_RT_BASE
  10440. + tsk->posix_timer_list = NULL;
  10441. +#endif
  10442. tsk->cputime_expires.prof_exp = 0;
  10443. tsk->cputime_expires.virt_exp = 0;
  10444. tsk->cputime_expires.sched_exp = 0;
  10445. @@ -1552,6 +1591,7 @@
  10446. spin_lock_init(&p->alloc_lock);
  10447. init_sigpending(&p->pending);
  10448. + p->sigqueue_cache = NULL;
  10449. p->utime = p->stime = p->gtime = 0;
  10450. p->utimescaled = p->stimescaled = 0;
  10451. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/futex.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/futex.c
  10452. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/futex.c 2017-04-16 10:38:29.000000000 +0200
  10453. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/futex.c 2017-04-18 17:54:26.000000000 +0200
  10454. @@ -800,7 +800,7 @@
  10455. return 0;
  10456. }
  10457. -static struct futex_pi_state * alloc_pi_state(void)
  10458. +static struct futex_pi_state *alloc_pi_state(void)
  10459. {
  10460. struct futex_pi_state *pi_state = current->pi_state_cache;
  10461. @@ -810,6 +810,11 @@
  10462. return pi_state;
  10463. }
  10464. +static void get_pi_state(struct futex_pi_state *pi_state)
  10465. +{
  10466. + WARN_ON_ONCE(!atomic_inc_not_zero(&pi_state->refcount));
  10467. +}
  10468. +
  10469. /*
  10470. * Drops a reference to the pi_state object and frees or caches it
  10471. * when the last reference is gone.
  10472. @@ -854,7 +859,7 @@
  10473. * Look up the task based on what TID userspace gave us.
  10474. * We dont trust it.
  10475. */
  10476. -static struct task_struct * futex_find_get_task(pid_t pid)
  10477. +static struct task_struct *futex_find_get_task(pid_t pid)
  10478. {
  10479. struct task_struct *p;
  10480. @@ -904,7 +909,9 @@
  10481. * task still owns the PI-state:
  10482. */
  10483. if (head->next != next) {
  10484. + raw_spin_unlock_irq(&curr->pi_lock);
  10485. spin_unlock(&hb->lock);
  10486. + raw_spin_lock_irq(&curr->pi_lock);
  10487. continue;
  10488. }
  10489. @@ -914,10 +921,12 @@
  10490. pi_state->owner = NULL;
  10491. raw_spin_unlock_irq(&curr->pi_lock);
  10492. - rt_mutex_unlock(&pi_state->pi_mutex);
  10493. -
  10494. + get_pi_state(pi_state);
  10495. spin_unlock(&hb->lock);
  10496. + rt_mutex_futex_unlock(&pi_state->pi_mutex);
  10497. + put_pi_state(pi_state);
  10498. +
  10499. raw_spin_lock_irq(&curr->pi_lock);
  10500. }
  10501. raw_spin_unlock_irq(&curr->pi_lock);
  10502. @@ -971,6 +980,39 @@
  10503. *
  10504. * [10] There is no transient state which leaves owner and user space
  10505. * TID out of sync.
  10506. + *
  10507. + *
  10508. + * Serialization and lifetime rules:
  10509. + *
  10510. + * hb->lock:
  10511. + *
  10512. + * hb -> futex_q, relation
  10513. + * futex_q -> pi_state, relation
  10514. + *
  10515. + * (cannot be raw because hb can contain arbitrary amount
  10516. + * of futex_q's)
  10517. + *
  10518. + * pi_mutex->wait_lock:
  10519. + *
  10520. + * {uval, pi_state}
  10521. + *
  10522. + * (and pi_mutex 'obviously')
  10523. + *
  10524. + * p->pi_lock:
  10525. + *
  10526. + * p->pi_state_list -> pi_state->list, relation
  10527. + *
  10528. + * pi_state->refcount:
  10529. + *
  10530. + * pi_state lifetime
  10531. + *
  10532. + *
  10533. + * Lock order:
  10534. + *
  10535. + * hb->lock
  10536. + * pi_mutex->wait_lock
  10537. + * p->pi_lock
  10538. + *
  10539. */
  10540. /*
  10541. @@ -978,10 +1020,12 @@
  10542. * the pi_state against the user space value. If correct, attach to
  10543. * it.
  10544. */
  10545. -static int attach_to_pi_state(u32 uval, struct futex_pi_state *pi_state,
  10546. +static int attach_to_pi_state(u32 __user *uaddr, u32 uval,
  10547. + struct futex_pi_state *pi_state,
  10548. struct futex_pi_state **ps)
  10549. {
  10550. pid_t pid = uval & FUTEX_TID_MASK;
  10551. + int ret, uval2;
  10552. /*
  10553. * Userspace might have messed up non-PI and PI futexes [3]
  10554. @@ -989,9 +1033,39 @@
  10555. if (unlikely(!pi_state))
  10556. return -EINVAL;
  10557. + /*
  10558. + * We get here with hb->lock held, and having found a
  10559. + * futex_top_waiter(). This means that futex_lock_pi() of said futex_q
  10560. + * has dropped the hb->lock in between queue_me() and unqueue_me_pi(),
  10561. + * which in turn means that futex_lock_pi() still has a reference on
  10562. + * our pi_state.
  10563. + *
  10564. + * The waiter holding a reference on @pi_state also protects against
  10565. + * the unlocked put_pi_state() in futex_unlock_pi(), futex_lock_pi()
  10566. + * and futex_wait_requeue_pi() as it cannot go to 0 and consequently
  10567. + * free pi_state before we can take a reference ourselves.
  10568. + */
  10569. WARN_ON(!atomic_read(&pi_state->refcount));
  10570. /*
  10571. + * Now that we have a pi_state, we can acquire wait_lock
  10572. + * and do the state validation.
  10573. + */
  10574. + raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
  10575. +
  10576. + /*
  10577. + * Since {uval, pi_state} is serialized by wait_lock, and our current
  10578. + * uval was read without holding it, it can have changed. Verify it
  10579. + * still is what we expect it to be, otherwise retry the entire
  10580. + * operation.
  10581. + */
  10582. + if (get_futex_value_locked(&uval2, uaddr))
  10583. + goto out_efault;
  10584. +
  10585. + if (uval != uval2)
  10586. + goto out_eagain;
  10587. +
  10588. + /*
  10589. * Handle the owner died case:
  10590. */
  10591. if (uval & FUTEX_OWNER_DIED) {
  10592. @@ -1006,11 +1080,11 @@
  10593. * is not 0. Inconsistent state. [5]
  10594. */
  10595. if (pid)
  10596. - return -EINVAL;
  10597. + goto out_einval;
  10598. /*
  10599. * Take a ref on the state and return success. [4]
  10600. */
  10601. - goto out_state;
  10602. + goto out_attach;
  10603. }
  10604. /*
  10605. @@ -1022,14 +1096,14 @@
  10606. * Take a ref on the state and return success. [6]
  10607. */
  10608. if (!pid)
  10609. - goto out_state;
  10610. + goto out_attach;
  10611. } else {
  10612. /*
  10613. * If the owner died bit is not set, then the pi_state
  10614. * must have an owner. [7]
  10615. */
  10616. if (!pi_state->owner)
  10617. - return -EINVAL;
  10618. + goto out_einval;
  10619. }
  10620. /*
  10621. @@ -1038,11 +1112,29 @@
  10622. * user space TID. [9/10]
  10623. */
  10624. if (pid != task_pid_vnr(pi_state->owner))
  10625. - return -EINVAL;
  10626. -out_state:
  10627. - atomic_inc(&pi_state->refcount);
  10628. + goto out_einval;
  10629. +
  10630. +out_attach:
  10631. + get_pi_state(pi_state);
  10632. + raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
  10633. *ps = pi_state;
  10634. return 0;
  10635. +
  10636. +out_einval:
  10637. + ret = -EINVAL;
  10638. + goto out_error;
  10639. +
  10640. +out_eagain:
  10641. + ret = -EAGAIN;
  10642. + goto out_error;
  10643. +
  10644. +out_efault:
  10645. + ret = -EFAULT;
  10646. + goto out_error;
  10647. +
  10648. +out_error:
  10649. + raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
  10650. + return ret;
  10651. }
  10652. /*
  10653. @@ -1093,6 +1185,9 @@
  10654. /*
  10655. * No existing pi state. First waiter. [2]
  10656. + *
  10657. + * This creates pi_state, we have hb->lock held, this means nothing can
  10658. + * observe this state, wait_lock is irrelevant.
  10659. */
  10660. pi_state = alloc_pi_state();
  10661. @@ -1117,17 +1212,18 @@
  10662. return 0;
  10663. }
  10664. -static int lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
  10665. +static int lookup_pi_state(u32 __user *uaddr, u32 uval,
  10666. + struct futex_hash_bucket *hb,
  10667. union futex_key *key, struct futex_pi_state **ps)
  10668. {
  10669. - struct futex_q *match = futex_top_waiter(hb, key);
  10670. + struct futex_q *top_waiter = futex_top_waiter(hb, key);
  10671. /*
  10672. * If there is a waiter on that futex, validate it and
  10673. * attach to the pi_state when the validation succeeds.
  10674. */
  10675. - if (match)
  10676. - return attach_to_pi_state(uval, match->pi_state, ps);
  10677. + if (top_waiter)
  10678. + return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);
  10679. /*
  10680. * We are the first waiter - try to look up the owner based on
  10681. @@ -1146,7 +1242,7 @@
  10682. if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)))
  10683. return -EFAULT;
  10684. - /*If user space value changed, let the caller retry */
  10685. + /* If user space value changed, let the caller retry */
  10686. return curval != uval ? -EAGAIN : 0;
  10687. }
  10688. @@ -1174,7 +1270,7 @@
  10689. struct task_struct *task, int set_waiters)
  10690. {
  10691. u32 uval, newval, vpid = task_pid_vnr(task);
  10692. - struct futex_q *match;
  10693. + struct futex_q *top_waiter;
  10694. int ret;
  10695. /*
  10696. @@ -1200,9 +1296,9 @@
  10697. * Lookup existing state first. If it exists, try to attach to
  10698. * its pi_state.
  10699. */
  10700. - match = futex_top_waiter(hb, key);
  10701. - if (match)
  10702. - return attach_to_pi_state(uval, match->pi_state, ps);
  10703. + top_waiter = futex_top_waiter(hb, key);
  10704. + if (top_waiter)
  10705. + return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);
  10706. /*
  10707. * No waiter and user TID is 0. We are here because the
  10708. @@ -1288,45 +1384,39 @@
  10709. * memory barrier is required here to prevent the following
  10710. * store to lock_ptr from getting ahead of the plist_del.
  10711. */
  10712. - smp_wmb();
  10713. - q->lock_ptr = NULL;
  10714. + smp_store_release(&q->lock_ptr, NULL);
  10715. }
  10716. -static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this,
  10717. - struct futex_hash_bucket *hb)
  10718. +/*
  10719. + * Caller must hold a reference on @pi_state.
  10720. + */
  10721. +static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state)
  10722. {
  10723. - struct task_struct *new_owner;
  10724. - struct futex_pi_state *pi_state = this->pi_state;
  10725. u32 uninitialized_var(curval), newval;
  10726. + struct task_struct *new_owner;
  10727. + bool deboost = false;
  10728. WAKE_Q(wake_q);
  10729. - bool deboost;
  10730. + WAKE_Q(wake_sleeper_q);
  10731. int ret = 0;
  10732. - if (!pi_state)
  10733. - return -EINVAL;
  10734. -
  10735. - /*
  10736. - * If current does not own the pi_state then the futex is
  10737. - * inconsistent and user space fiddled with the futex value.
  10738. - */
  10739. - if (pi_state->owner != current)
  10740. - return -EINVAL;
  10741. -
  10742. - raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
  10743. new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);
  10744. + if (WARN_ON_ONCE(!new_owner)) {
  10745. + /*
  10746. + * As per the comment in futex_unlock_pi() this should not happen.
  10747. + *
  10748. + * When this happens, give up our locks and try again, giving
  10749. + * the futex_lock_pi() instance time to complete, either by
  10750. + * waiting on the rtmutex or removing itself from the futex
  10751. + * queue.
  10752. + */
  10753. + ret = -EAGAIN;
  10754. + goto out_unlock;
  10755. + }
  10756. /*
  10757. - * It is possible that the next waiter (the one that brought
  10758. - * this owner to the kernel) timed out and is no longer
  10759. - * waiting on the lock.
  10760. - */
  10761. - if (!new_owner)
  10762. - new_owner = this->task;
  10763. -
  10764. - /*
  10765. - * We pass it to the next owner. The WAITERS bit is always
  10766. - * kept enabled while there is PI state around. We cleanup the
  10767. - * owner died bit, because we are the owner.
  10768. + * We pass it to the next owner. The WAITERS bit is always kept
  10769. + * enabled while there is PI state around. We cleanup the owner
  10770. + * died bit, because we are the owner.
  10771. */
  10772. newval = FUTEX_WAITERS | task_pid_vnr(new_owner);
  10773. @@ -1335,6 +1425,7 @@
  10774. if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)) {
  10775. ret = -EFAULT;
  10776. +
  10777. } else if (curval != uval) {
  10778. /*
  10779. * If a unconditional UNLOCK_PI operation (user space did not
  10780. @@ -1347,10 +1438,9 @@
  10781. else
  10782. ret = -EINVAL;
  10783. }
  10784. - if (ret) {
  10785. - raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
  10786. - return ret;
  10787. - }
  10788. +
  10789. + if (ret)
  10790. + goto out_unlock;
  10791. raw_spin_lock(&pi_state->owner->pi_lock);
  10792. WARN_ON(list_empty(&pi_state->list));
  10793. @@ -1363,22 +1453,22 @@
  10794. pi_state->owner = new_owner;
  10795. raw_spin_unlock(&new_owner->pi_lock);
  10796. - raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
  10797. -
  10798. - deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);
  10799. -
  10800. /*
  10801. - * First unlock HB so the waiter does not spin on it once he got woken
  10802. - * up. Second wake up the waiter before the priority is adjusted. If we
  10803. - * deboost first (and lose our higher priority), then the task might get
  10804. - * scheduled away before the wake up can take place.
  10805. + * We've updated the uservalue, this unlock cannot fail.
  10806. */
  10807. - spin_unlock(&hb->lock);
  10808. - wake_up_q(&wake_q);
  10809. - if (deboost)
  10810. + deboost = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q,
  10811. + &wake_sleeper_q);
  10812. +
  10813. +out_unlock:
  10814. + raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
  10815. +
  10816. + if (deboost) {
  10817. + wake_up_q(&wake_q);
  10818. + wake_up_q_sleeper(&wake_sleeper_q);
  10819. rt_mutex_adjust_prio(current);
  10820. + }
  10821. - return 0;
  10822. + return ret;
  10823. }
  10824. /*
  10825. @@ -1824,7 +1914,7 @@
  10826. * If that call succeeds then we have pi_state and an
  10827. * initial refcount on it.
  10828. */
  10829. - ret = lookup_pi_state(ret, hb2, &key2, &pi_state);
  10830. + ret = lookup_pi_state(uaddr2, ret, hb2, &key2, &pi_state);
  10831. }
  10832. switch (ret) {
  10833. @@ -1907,7 +1997,7 @@
  10834. * refcount on the pi_state and store the pointer in
  10835. * the futex_q object of the waiter.
  10836. */
  10837. - atomic_inc(&pi_state->refcount);
  10838. + get_pi_state(pi_state);
  10839. this->pi_state = pi_state;
  10840. ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,
  10841. this->rt_waiter,
  10842. @@ -1924,6 +2014,16 @@
  10843. requeue_pi_wake_futex(this, &key2, hb2);
  10844. drop_count++;
  10845. continue;
  10846. + } else if (ret == -EAGAIN) {
  10847. + /*
  10848. + * Waiter was woken by timeout or
  10849. + * signal and has set pi_blocked_on to
  10850. + * PI_WAKEUP_INPROGRESS before we
  10851. + * tried to enqueue it on the rtmutex.
  10852. + */
  10853. + this->pi_state = NULL;
  10854. + put_pi_state(pi_state);
  10855. + continue;
  10856. } else if (ret) {
  10857. /*
  10858. * rt_mutex_start_proxy_lock() detected a
  10859. @@ -2007,20 +2107,7 @@
  10860. hb_waiters_dec(hb);
  10861. }
  10862. -/**
  10863. - * queue_me() - Enqueue the futex_q on the futex_hash_bucket
  10864. - * @q: The futex_q to enqueue
  10865. - * @hb: The destination hash bucket
  10866. - *
  10867. - * The hb->lock must be held by the caller, and is released here. A call to
  10868. - * queue_me() is typically paired with exactly one call to unqueue_me(). The
  10869. - * exceptions involve the PI related operations, which may use unqueue_me_pi()
  10870. - * or nothing if the unqueue is done as part of the wake process and the unqueue
  10871. - * state is implicit in the state of woken task (see futex_wait_requeue_pi() for
  10872. - * an example).
  10873. - */
  10874. -static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb)
  10875. - __releases(&hb->lock)
  10876. +static inline void __queue_me(struct futex_q *q, struct futex_hash_bucket *hb)
  10877. {
  10878. int prio;
  10879. @@ -2037,6 +2124,24 @@
  10880. plist_node_init(&q->list, prio);
  10881. plist_add(&q->list, &hb->chain);
  10882. q->task = current;
  10883. +}
  10884. +
  10885. +/**
  10886. + * queue_me() - Enqueue the futex_q on the futex_hash_bucket
  10887. + * @q: The futex_q to enqueue
  10888. + * @hb: The destination hash bucket
  10889. + *
  10890. + * The hb->lock must be held by the caller, and is released here. A call to
  10891. + * queue_me() is typically paired with exactly one call to unqueue_me(). The
  10892. + * exceptions involve the PI related operations, which may use unqueue_me_pi()
  10893. + * or nothing if the unqueue is done as part of the wake process and the unqueue
  10894. + * state is implicit in the state of woken task (see futex_wait_requeue_pi() for
  10895. + * an example).
  10896. + */
  10897. +static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb)
  10898. + __releases(&hb->lock)
  10899. +{
  10900. + __queue_me(q, hb);
  10901. spin_unlock(&hb->lock);
  10902. }
  10903. @@ -2123,10 +2228,13 @@
  10904. {
  10905. u32 newtid = task_pid_vnr(newowner) | FUTEX_WAITERS;
  10906. struct futex_pi_state *pi_state = q->pi_state;
  10907. - struct task_struct *oldowner = pi_state->owner;
  10908. u32 uval, uninitialized_var(curval), newval;
  10909. + struct task_struct *oldowner;
  10910. int ret;
  10911. + raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
  10912. +
  10913. + oldowner = pi_state->owner;
  10914. /* Owner died? */
  10915. if (!pi_state->owner)
  10916. newtid |= FUTEX_OWNER_DIED;
  10917. @@ -2134,7 +2242,8 @@
  10918. /*
  10919. * We are here either because we stole the rtmutex from the
  10920. * previous highest priority waiter or we are the highest priority
  10921. - * waiter but failed to get the rtmutex the first time.
  10922. + * waiter but have failed to get the rtmutex the first time.
  10923. + *
  10924. * We have to replace the newowner TID in the user space variable.
  10925. * This must be atomic as we have to preserve the owner died bit here.
  10926. *
  10927. @@ -2142,17 +2251,16 @@
  10928. * because we can fault here. Imagine swapped out pages or a fork
  10929. * that marked all the anonymous memory readonly for cow.
  10930. *
  10931. - * Modifying pi_state _before_ the user space value would
  10932. - * leave the pi_state in an inconsistent state when we fault
  10933. - * here, because we need to drop the hash bucket lock to
  10934. - * handle the fault. This might be observed in the PID check
  10935. - * in lookup_pi_state.
  10936. + * Modifying pi_state _before_ the user space value would leave the
  10937. + * pi_state in an inconsistent state when we fault here, because we
  10938. + * need to drop the locks to handle the fault. This might be observed
  10939. + * in the PID check in lookup_pi_state.
  10940. */
  10941. retry:
  10942. if (get_futex_value_locked(&uval, uaddr))
  10943. goto handle_fault;
  10944. - while (1) {
  10945. + for (;;) {
  10946. newval = (uval & FUTEX_OWNER_DIED) | newtid;
  10947. if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval))
  10948. @@ -2167,47 +2275,60 @@
  10949. * itself.
  10950. */
  10951. if (pi_state->owner != NULL) {
  10952. - raw_spin_lock_irq(&pi_state->owner->pi_lock);
  10953. + raw_spin_lock(&pi_state->owner->pi_lock);
  10954. WARN_ON(list_empty(&pi_state->list));
  10955. list_del_init(&pi_state->list);
  10956. - raw_spin_unlock_irq(&pi_state->owner->pi_lock);
  10957. + raw_spin_unlock(&pi_state->owner->pi_lock);
  10958. }
  10959. pi_state->owner = newowner;
  10960. - raw_spin_lock_irq(&newowner->pi_lock);
  10961. + raw_spin_lock(&newowner->pi_lock);
  10962. WARN_ON(!list_empty(&pi_state->list));
  10963. list_add(&pi_state->list, &newowner->pi_state_list);
  10964. - raw_spin_unlock_irq(&newowner->pi_lock);
  10965. + raw_spin_unlock(&newowner->pi_lock);
  10966. + raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
  10967. +
  10968. return 0;
  10969. /*
  10970. - * To handle the page fault we need to drop the hash bucket
  10971. - * lock here. That gives the other task (either the highest priority
  10972. - * waiter itself or the task which stole the rtmutex) the
  10973. - * chance to try the fixup of the pi_state. So once we are
  10974. - * back from handling the fault we need to check the pi_state
  10975. - * after reacquiring the hash bucket lock and before trying to
  10976. - * do another fixup. When the fixup has been done already we
  10977. - * simply return.
  10978. + * To handle the page fault we need to drop the locks here. That gives
  10979. + * the other task (either the highest priority waiter itself or the
  10980. + * task which stole the rtmutex) the chance to try the fixup of the
  10981. + * pi_state. So once we are back from handling the fault we need to
  10982. + * check the pi_state after reacquiring the locks and before trying to
  10983. + * do another fixup. When the fixup has been done already we simply
  10984. + * return.
  10985. + *
  10986. + * Note: we hold both hb->lock and pi_mutex->wait_lock. We can safely
  10987. + * drop hb->lock since the caller owns the hb -> futex_q relation.
  10988. + * Dropping the pi_mutex->wait_lock requires the state revalidate.
  10989. */
  10990. handle_fault:
  10991. + raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
  10992. spin_unlock(q->lock_ptr);
  10993. ret = fault_in_user_writeable(uaddr);
  10994. spin_lock(q->lock_ptr);
  10995. + raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
  10996. /*
  10997. * Check if someone else fixed it for us:
  10998. */
  10999. - if (pi_state->owner != oldowner)
  11000. - return 0;
  11001. + if (pi_state->owner != oldowner) {
  11002. + ret = 0;
  11003. + goto out_unlock;
  11004. + }
  11005. if (ret)
  11006. - return ret;
  11007. + goto out_unlock;
  11008. goto retry;
  11009. +
  11010. +out_unlock:
  11011. + raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
  11012. + return ret;
  11013. }
  11014. static long futex_wait_restart(struct restart_block *restart);
  11015. @@ -2229,13 +2350,16 @@
  11016. */
  11017. static int fixup_owner(u32 __user *uaddr, struct futex_q *q, int locked)
  11018. {
  11019. - struct task_struct *owner;
  11020. int ret = 0;
  11021. if (locked) {
  11022. /*
  11023. * Got the lock. We might not be the anticipated owner if we
  11024. * did a lock-steal - fix up the PI-state in that case:
  11025. + *
  11026. + * We can safely read pi_state->owner without holding wait_lock
  11027. + * because we now own the rt_mutex, only the owner will attempt
  11028. + * to change it.
  11029. */
  11030. if (q->pi_state->owner != current)
  11031. ret = fixup_pi_state_owner(uaddr, q, current);
  11032. @@ -2243,43 +2367,15 @@
  11033. }
  11034. /*
  11035. - * Catch the rare case, where the lock was released when we were on the
  11036. - * way back before we locked the hash bucket.
  11037. - */
  11038. - if (q->pi_state->owner == current) {
  11039. - /*
  11040. - * Try to get the rt_mutex now. This might fail as some other
  11041. - * task acquired the rt_mutex after we removed ourself from the
  11042. - * rt_mutex waiters list.
  11043. - */
  11044. - if (rt_mutex_trylock(&q->pi_state->pi_mutex)) {
  11045. - locked = 1;
  11046. - goto out;
  11047. - }
  11048. -
  11049. - /*
  11050. - * pi_state is incorrect, some other task did a lock steal and
  11051. - * we returned due to timeout or signal without taking the
  11052. - * rt_mutex. Too late.
  11053. - */
  11054. - raw_spin_lock_irq(&q->pi_state->pi_mutex.wait_lock);
  11055. - owner = rt_mutex_owner(&q->pi_state->pi_mutex);
  11056. - if (!owner)
  11057. - owner = rt_mutex_next_owner(&q->pi_state->pi_mutex);
  11058. - raw_spin_unlock_irq(&q->pi_state->pi_mutex.wait_lock);
  11059. - ret = fixup_pi_state_owner(uaddr, q, owner);
  11060. - goto out;
  11061. - }
  11062. -
  11063. - /*
  11064. * Paranoia check. If we did not take the lock, then we should not be
  11065. * the owner of the rt_mutex.
  11066. */
  11067. - if (rt_mutex_owner(&q->pi_state->pi_mutex) == current)
  11068. + if (rt_mutex_owner(&q->pi_state->pi_mutex) == current) {
  11069. printk(KERN_ERR "fixup_owner: ret = %d pi-mutex: %p "
  11070. "pi-state %p\n", ret,
  11071. q->pi_state->pi_mutex.owner,
  11072. q->pi_state->owner);
  11073. + }
  11074. out:
  11075. return ret ? ret : locked;
  11076. @@ -2503,6 +2599,8 @@
  11077. ktime_t *time, int trylock)
  11078. {
  11079. struct hrtimer_sleeper timeout, *to = NULL;
  11080. + struct futex_pi_state *pi_state = NULL;
  11081. + struct rt_mutex_waiter rt_waiter;
  11082. struct futex_hash_bucket *hb;
  11083. struct futex_q q = futex_q_init;
  11084. int res, ret;
  11085. @@ -2555,25 +2653,77 @@
  11086. }
  11087. }
  11088. + WARN_ON(!q.pi_state);
  11089. +
  11090. /*
  11091. * Only actually queue now that the atomic ops are done:
  11092. */
  11093. - queue_me(&q, hb);
  11094. + __queue_me(&q, hb);
  11095. - WARN_ON(!q.pi_state);
  11096. - /*
  11097. - * Block on the PI mutex:
  11098. - */
  11099. - if (!trylock) {
  11100. - ret = rt_mutex_timed_futex_lock(&q.pi_state->pi_mutex, to);
  11101. - } else {
  11102. - ret = rt_mutex_trylock(&q.pi_state->pi_mutex);
  11103. + if (trylock) {
  11104. + ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);
  11105. /* Fixup the trylock return value: */
  11106. ret = ret ? 0 : -EWOULDBLOCK;
  11107. + goto no_block;
  11108. + }
  11109. +
  11110. + rt_mutex_init_waiter(&rt_waiter, false);
  11111. +
  11112. + /*
  11113. + * On PREEMPT_RT_FULL, when hb->lock becomes an rt_mutex, we must not
  11114. + * hold it while doing rt_mutex_start_proxy(), because then it will
  11115. + * include hb->lock in the blocking chain, even through we'll not in
  11116. + * fact hold it while blocking. This will lead it to report -EDEADLK
  11117. + * and BUG when futex_unlock_pi() interleaves with this.
  11118. + *
  11119. + * Therefore acquire wait_lock while holding hb->lock, but drop the
  11120. + * latter before calling rt_mutex_start_proxy_lock(). This still fully
  11121. + * serializes against futex_unlock_pi() as that does the exact same
  11122. + * lock handoff sequence.
  11123. + */
  11124. + raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);
  11125. + /*
  11126. + * the migrate_disable() here disables migration in the in_atomic() fast
  11127. + * path which is enabled again in the following spin_unlock(). We have
  11128. + * one migrate_disable() pending in the slow-path which is reversed
  11129. + * after the raw_spin_unlock_irq() where we leave the atomic context.
  11130. + */
  11131. + migrate_disable();
  11132. +
  11133. + spin_unlock(q.lock_ptr);
  11134. + ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current);
  11135. + raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock);
  11136. + migrate_enable();
  11137. +
  11138. + if (ret) {
  11139. + if (ret == 1)
  11140. + ret = 0;
  11141. +
  11142. + spin_lock(q.lock_ptr);
  11143. + goto no_block;
  11144. }
  11145. +
  11146. + if (unlikely(to))
  11147. + hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
  11148. +
  11149. + ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
  11150. +
  11151. spin_lock(q.lock_ptr);
  11152. /*
  11153. + * If we failed to acquire the lock (signal/timeout), we must
  11154. + * first acquire the hb->lock before removing the lock from the
  11155. + * rt_mutex waitqueue, such that we can keep the hb and rt_mutex
  11156. + * wait lists consistent.
  11157. + *
  11158. + * In particular; it is important that futex_unlock_pi() can not
  11159. + * observe this inconsistency.
  11160. + */
  11161. + if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))
  11162. + ret = 0;
  11163. +
  11164. +no_block:
  11165. + /*
  11166. * Fixup the pi_state owner and possibly acquire the lock if we
  11167. * haven't already.
  11168. */
  11169. @@ -2589,12 +2739,19 @@
  11170. * If fixup_owner() faulted and was unable to handle the fault, unlock
  11171. * it and return the fault to userspace.
  11172. */
  11173. - if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current))
  11174. - rt_mutex_unlock(&q.pi_state->pi_mutex);
  11175. + if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current)) {
  11176. + pi_state = q.pi_state;
  11177. + get_pi_state(pi_state);
  11178. + }
  11179. /* Unqueue and drop the lock */
  11180. unqueue_me_pi(&q);
  11181. + if (pi_state) {
  11182. + rt_mutex_futex_unlock(&pi_state->pi_mutex);
  11183. + put_pi_state(pi_state);
  11184. + }
  11185. +
  11186. goto out_put_key;
  11187. out_unlock_put_key:
  11188. @@ -2631,7 +2788,7 @@
  11189. u32 uninitialized_var(curval), uval, vpid = task_pid_vnr(current);
  11190. union futex_key key = FUTEX_KEY_INIT;
  11191. struct futex_hash_bucket *hb;
  11192. - struct futex_q *match;
  11193. + struct futex_q *top_waiter;
  11194. int ret;
  11195. retry:
  11196. @@ -2655,12 +2812,48 @@
  11197. * all and we at least want to know if user space fiddled
  11198. * with the futex value instead of blindly unlocking.
  11199. */
  11200. - match = futex_top_waiter(hb, &key);
  11201. - if (match) {
  11202. - ret = wake_futex_pi(uaddr, uval, match, hb);
  11203. + top_waiter = futex_top_waiter(hb, &key);
  11204. + if (top_waiter) {
  11205. + struct futex_pi_state *pi_state = top_waiter->pi_state;
  11206. +
  11207. + ret = -EINVAL;
  11208. + if (!pi_state)
  11209. + goto out_unlock;
  11210. +
  11211. /*
  11212. - * In case of success wake_futex_pi dropped the hash
  11213. - * bucket lock.
  11214. + * If current does not own the pi_state then the futex is
  11215. + * inconsistent and user space fiddled with the futex value.
  11216. + */
  11217. + if (pi_state->owner != current)
  11218. + goto out_unlock;
  11219. +
  11220. + get_pi_state(pi_state);
  11221. + /*
  11222. + * By taking wait_lock while still holding hb->lock, we ensure
  11223. + * there is no point where we hold neither; and therefore
  11224. + * wake_futex_pi() must observe a state consistent with what we
  11225. + * observed.
  11226. + */
  11227. + raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
  11228. + /*
  11229. + * Magic trickery for now to make the RT migrate disable
  11230. + * logic happy. The following spin_unlock() happens with
  11231. + * interrupts disabled so the internal migrate_enable()
  11232. + * won't undo the migrate_disable() which was issued when
  11233. + * locking hb->lock.
  11234. + */
  11235. + migrate_disable();
  11236. + spin_unlock(&hb->lock);
  11237. +
  11238. + /* Drops pi_state->pi_mutex.wait_lock */
  11239. + ret = wake_futex_pi(uaddr, uval, pi_state);
  11240. +
  11241. + migrate_enable();
  11242. +
  11243. + put_pi_state(pi_state);
  11244. +
  11245. + /*
  11246. + * Success, we're done! No tricky corner cases.
  11247. */
  11248. if (!ret)
  11249. goto out_putkey;
  11250. @@ -2675,7 +2868,6 @@
  11251. * setting the FUTEX_WAITERS bit. Try again.
  11252. */
  11253. if (ret == -EAGAIN) {
  11254. - spin_unlock(&hb->lock);
  11255. put_futex_key(&key);
  11256. goto retry;
  11257. }
  11258. @@ -2683,7 +2875,7 @@
  11259. * wake_futex_pi has detected invalid state. Tell user
  11260. * space.
  11261. */
  11262. - goto out_unlock;
  11263. + goto out_putkey;
  11264. }
  11265. /*
  11266. @@ -2693,8 +2885,10 @@
  11267. * preserve the WAITERS bit not the OWNER_DIED one. We are the
  11268. * owner.
  11269. */
  11270. - if (cmpxchg_futex_value_locked(&curval, uaddr, uval, 0))
  11271. + if (cmpxchg_futex_value_locked(&curval, uaddr, uval, 0)) {
  11272. + spin_unlock(&hb->lock);
  11273. goto pi_faulted;
  11274. + }
  11275. /*
  11276. * If uval has changed, let user space handle it.
  11277. @@ -2708,7 +2902,6 @@
  11278. return ret;
  11279. pi_faulted:
  11280. - spin_unlock(&hb->lock);
  11281. put_futex_key(&key);
  11282. ret = fault_in_user_writeable(uaddr);
  11283. @@ -2812,8 +3005,9 @@
  11284. u32 __user *uaddr2)
  11285. {
  11286. struct hrtimer_sleeper timeout, *to = NULL;
  11287. + struct futex_pi_state *pi_state = NULL;
  11288. struct rt_mutex_waiter rt_waiter;
  11289. - struct futex_hash_bucket *hb;
  11290. + struct futex_hash_bucket *hb, *hb2;
  11291. union futex_key key2 = FUTEX_KEY_INIT;
  11292. struct futex_q q = futex_q_init;
  11293. int res, ret;
  11294. @@ -2838,10 +3032,7 @@
  11295. * The waiter is allocated on our stack, manipulated by the requeue
  11296. * code while we sleep on uaddr.
  11297. */
  11298. - debug_rt_mutex_init_waiter(&rt_waiter);
  11299. - RB_CLEAR_NODE(&rt_waiter.pi_tree_entry);
  11300. - RB_CLEAR_NODE(&rt_waiter.tree_entry);
  11301. - rt_waiter.task = NULL;
  11302. + rt_mutex_init_waiter(&rt_waiter, false);
  11303. ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE);
  11304. if (unlikely(ret != 0))
  11305. @@ -2872,20 +3063,55 @@
  11306. /* Queue the futex_q, drop the hb lock, wait for wakeup. */
  11307. futex_wait_queue_me(hb, &q, to);
  11308. - spin_lock(&hb->lock);
  11309. - ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to);
  11310. - spin_unlock(&hb->lock);
  11311. - if (ret)
  11312. - goto out_put_keys;
  11313. + /*
  11314. + * On RT we must avoid races with requeue and trying to block
  11315. + * on two mutexes (hb->lock and uaddr2's rtmutex) by
  11316. + * serializing access to pi_blocked_on with pi_lock.
  11317. + */
  11318. + raw_spin_lock_irq(&current->pi_lock);
  11319. + if (current->pi_blocked_on) {
  11320. + /*
  11321. + * We have been requeued or are in the process of
  11322. + * being requeued.
  11323. + */
  11324. + raw_spin_unlock_irq(&current->pi_lock);
  11325. + } else {
  11326. + /*
  11327. + * Setting pi_blocked_on to PI_WAKEUP_INPROGRESS
  11328. + * prevents a concurrent requeue from moving us to the
  11329. + * uaddr2 rtmutex. After that we can safely acquire
  11330. + * (and possibly block on) hb->lock.
  11331. + */
  11332. + current->pi_blocked_on = PI_WAKEUP_INPROGRESS;
  11333. + raw_spin_unlock_irq(&current->pi_lock);
  11334. +
  11335. + spin_lock(&hb->lock);
  11336. +
  11337. + /*
  11338. + * Clean up pi_blocked_on. We might leak it otherwise
  11339. + * when we succeeded with the hb->lock in the fast
  11340. + * path.
  11341. + */
  11342. + raw_spin_lock_irq(&current->pi_lock);
  11343. + current->pi_blocked_on = NULL;
  11344. + raw_spin_unlock_irq(&current->pi_lock);
  11345. +
  11346. + ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to);
  11347. + spin_unlock(&hb->lock);
  11348. + if (ret)
  11349. + goto out_put_keys;
  11350. + }
  11351. /*
  11352. - * In order for us to be here, we know our q.key == key2, and since
  11353. - * we took the hb->lock above, we also know that futex_requeue() has
  11354. - * completed and we no longer have to concern ourselves with a wakeup
  11355. - * race with the atomic proxy lock acquisition by the requeue code. The
  11356. - * futex_requeue dropped our key1 reference and incremented our key2
  11357. - * reference count.
  11358. + * In order to be here, we have either been requeued, are in
  11359. + * the process of being requeued, or requeue successfully
  11360. + * acquired uaddr2 on our behalf. If pi_blocked_on was
  11361. + * non-null above, we may be racing with a requeue. Do not
  11362. + * rely on q->lock_ptr to be hb2->lock until after blocking on
  11363. + * hb->lock or hb2->lock. The futex_requeue dropped our key1
  11364. + * reference and incremented our key2 reference count.
  11365. */
  11366. + hb2 = hash_futex(&key2);
  11367. /* Check if the requeue code acquired the second futex for us. */
  11368. if (!q.rt_waiter) {
  11369. @@ -2894,16 +3120,19 @@
  11370. * did a lock-steal - fix up the PI-state in that case.
  11371. */
  11372. if (q.pi_state && (q.pi_state->owner != current)) {
  11373. - spin_lock(q.lock_ptr);
  11374. + spin_lock(&hb2->lock);
  11375. + BUG_ON(&hb2->lock != q.lock_ptr);
  11376. ret = fixup_pi_state_owner(uaddr2, &q, current);
  11377. - if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current)
  11378. - rt_mutex_unlock(&q.pi_state->pi_mutex);
  11379. + if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) {
  11380. + pi_state = q.pi_state;
  11381. + get_pi_state(pi_state);
  11382. + }
  11383. /*
  11384. * Drop the reference to the pi state which
  11385. * the requeue_pi() code acquired for us.
  11386. */
  11387. put_pi_state(q.pi_state);
  11388. - spin_unlock(q.lock_ptr);
  11389. + spin_unlock(&hb2->lock);
  11390. }
  11391. } else {
  11392. struct rt_mutex *pi_mutex;
  11393. @@ -2915,10 +3144,14 @@
  11394. */
  11395. WARN_ON(!q.pi_state);
  11396. pi_mutex = &q.pi_state->pi_mutex;
  11397. - ret = rt_mutex_finish_proxy_lock(pi_mutex, to, &rt_waiter);
  11398. - debug_rt_mutex_free_waiter(&rt_waiter);
  11399. + ret = rt_mutex_wait_proxy_lock(pi_mutex, to, &rt_waiter);
  11400. - spin_lock(q.lock_ptr);
  11401. + spin_lock(&hb2->lock);
  11402. + BUG_ON(&hb2->lock != q.lock_ptr);
  11403. + if (ret && !rt_mutex_cleanup_proxy_lock(pi_mutex, &rt_waiter))
  11404. + ret = 0;
  11405. +
  11406. + debug_rt_mutex_free_waiter(&rt_waiter);
  11407. /*
  11408. * Fixup the pi_state owner and possibly acquire the lock if we
  11409. * haven't already.
  11410. @@ -2936,13 +3169,20 @@
  11411. * the fault, unlock the rt_mutex and return the fault to
  11412. * userspace.
  11413. */
  11414. - if (ret && rt_mutex_owner(pi_mutex) == current)
  11415. - rt_mutex_unlock(pi_mutex);
  11416. + if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) {
  11417. + pi_state = q.pi_state;
  11418. + get_pi_state(pi_state);
  11419. + }
  11420. /* Unqueue and drop the lock. */
  11421. unqueue_me_pi(&q);
  11422. }
  11423. + if (pi_state) {
  11424. + rt_mutex_futex_unlock(&pi_state->pi_mutex);
  11425. + put_pi_state(pi_state);
  11426. + }
  11427. +
  11428. if (ret == -EINTR) {
  11429. /*
  11430. * We've already been requeued, but cannot restart by calling
  11431. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/irq/handle.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/irq/handle.c
  11432. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/irq/handle.c 2017-04-16 10:38:29.000000000 +0200
  11433. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/irq/handle.c 2017-04-18 17:54:26.000000000 +0200
  11434. @@ -181,10 +181,16 @@
  11435. {
  11436. irqreturn_t retval;
  11437. unsigned int flags = 0;
  11438. + struct pt_regs *regs = get_irq_regs();
  11439. + u64 ip = regs ? instruction_pointer(regs) : 0;
  11440. retval = __handle_irq_event_percpu(desc, &flags);
  11441. - add_interrupt_randomness(desc->irq_data.irq, flags);
  11442. +#ifdef CONFIG_PREEMPT_RT_FULL
  11443. + desc->random_ip = ip;
  11444. +#else
  11445. + add_interrupt_randomness(desc->irq_data.irq, flags, ip);
  11446. +#endif
  11447. if (!noirqdebug)
  11448. note_interrupt(desc, retval);
  11449. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/irq/manage.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/irq/manage.c
  11450. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/irq/manage.c 2017-04-16 10:38:29.000000000 +0200
  11451. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/irq/manage.c 2017-04-18 17:54:26.000000000 +0200
  11452. @@ -22,6 +22,7 @@
  11453. #include "internals.h"
  11454. #ifdef CONFIG_IRQ_FORCED_THREADING
  11455. +# ifndef CONFIG_PREEMPT_RT_BASE
  11456. __read_mostly bool force_irqthreads;
  11457. static int __init setup_forced_irqthreads(char *arg)
  11458. @@ -30,6 +31,7 @@
  11459. return 0;
  11460. }
  11461. early_param("threadirqs", setup_forced_irqthreads);
  11462. +# endif
  11463. #endif
  11464. static void __synchronize_hardirq(struct irq_desc *desc)
  11465. @@ -233,7 +235,12 @@
  11466. if (desc->affinity_notify) {
  11467. kref_get(&desc->affinity_notify->kref);
  11468. +
  11469. +#ifdef CONFIG_PREEMPT_RT_BASE
  11470. + swork_queue(&desc->affinity_notify->swork);
  11471. +#else
  11472. schedule_work(&desc->affinity_notify->work);
  11473. +#endif
  11474. }
  11475. irqd_set(data, IRQD_AFFINITY_SET);
  11476. @@ -271,10 +278,8 @@
  11477. }
  11478. EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
  11479. -static void irq_affinity_notify(struct work_struct *work)
  11480. +static void _irq_affinity_notify(struct irq_affinity_notify *notify)
  11481. {
  11482. - struct irq_affinity_notify *notify =
  11483. - container_of(work, struct irq_affinity_notify, work);
  11484. struct irq_desc *desc = irq_to_desc(notify->irq);
  11485. cpumask_var_t cpumask;
  11486. unsigned long flags;
  11487. @@ -296,6 +301,35 @@
  11488. kref_put(&notify->kref, notify->release);
  11489. }
  11490. +#ifdef CONFIG_PREEMPT_RT_BASE
  11491. +static void init_helper_thread(void)
  11492. +{
  11493. + static int init_sworker_once;
  11494. +
  11495. + if (init_sworker_once)
  11496. + return;
  11497. + if (WARN_ON(swork_get()))
  11498. + return;
  11499. + init_sworker_once = 1;
  11500. +}
  11501. +
  11502. +static void irq_affinity_notify(struct swork_event *swork)
  11503. +{
  11504. + struct irq_affinity_notify *notify =
  11505. + container_of(swork, struct irq_affinity_notify, swork);
  11506. + _irq_affinity_notify(notify);
  11507. +}
  11508. +
  11509. +#else
  11510. +
  11511. +static void irq_affinity_notify(struct work_struct *work)
  11512. +{
  11513. + struct irq_affinity_notify *notify =
  11514. + container_of(work, struct irq_affinity_notify, work);
  11515. + _irq_affinity_notify(notify);
  11516. +}
  11517. +#endif
  11518. +
  11519. /**
  11520. * irq_set_affinity_notifier - control notification of IRQ affinity changes
  11521. * @irq: Interrupt for which to enable/disable notification
  11522. @@ -324,7 +358,12 @@
  11523. if (notify) {
  11524. notify->irq = irq;
  11525. kref_init(&notify->kref);
  11526. +#ifdef CONFIG_PREEMPT_RT_BASE
  11527. + INIT_SWORK(&notify->swork, irq_affinity_notify);
  11528. + init_helper_thread();
  11529. +#else
  11530. INIT_WORK(&notify->work, irq_affinity_notify);
  11531. +#endif
  11532. }
  11533. raw_spin_lock_irqsave(&desc->lock, flags);
  11534. @@ -879,7 +918,15 @@
  11535. local_bh_disable();
  11536. ret = action->thread_fn(action->irq, action->dev_id);
  11537. irq_finalize_oneshot(desc, action);
  11538. - local_bh_enable();
  11539. + /*
  11540. + * Interrupts which have real time requirements can be set up
  11541. + * to avoid softirq processing in the thread handler. This is
  11542. + * safe as these interrupts do not raise soft interrupts.
  11543. + */
  11544. + if (irq_settings_no_softirq_call(desc))
  11545. + _local_bh_enable();
  11546. + else
  11547. + local_bh_enable();
  11548. return ret;
  11549. }
  11550. @@ -976,6 +1023,12 @@
  11551. if (action_ret == IRQ_WAKE_THREAD)
  11552. irq_wake_secondary(desc, action);
  11553. +#ifdef CONFIG_PREEMPT_RT_FULL
  11554. + migrate_disable();
  11555. + add_interrupt_randomness(action->irq, 0,
  11556. + desc->random_ip ^ (unsigned long) action);
  11557. + migrate_enable();
  11558. +#endif
  11559. wake_threads_waitq(desc);
  11560. }
  11561. @@ -1336,6 +1389,9 @@
  11562. irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
  11563. }
  11564. + if (new->flags & IRQF_NO_SOFTIRQ_CALL)
  11565. + irq_settings_set_no_softirq_call(desc);
  11566. +
  11567. /* Set default affinity mask once everything is setup */
  11568. setup_affinity(desc, mask);
  11569. @@ -2061,7 +2117,7 @@
  11570. * This call sets the internal irqchip state of an interrupt,
  11571. * depending on the value of @which.
  11572. *
  11573. - * This function should be called with preemption disabled if the
  11574. + * This function should be called with migration disabled if the
  11575. * interrupt controller has per-cpu registers.
  11576. */
  11577. int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
  11578. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/irq/settings.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/irq/settings.h
  11579. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/irq/settings.h 2017-04-16 10:38:29.000000000 +0200
  11580. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/irq/settings.h 2017-04-18 17:54:26.000000000 +0200
  11581. @@ -16,6 +16,7 @@
  11582. _IRQ_PER_CPU_DEVID = IRQ_PER_CPU_DEVID,
  11583. _IRQ_IS_POLLED = IRQ_IS_POLLED,
  11584. _IRQ_DISABLE_UNLAZY = IRQ_DISABLE_UNLAZY,
  11585. + _IRQ_NO_SOFTIRQ_CALL = IRQ_NO_SOFTIRQ_CALL,
  11586. _IRQF_MODIFY_MASK = IRQF_MODIFY_MASK,
  11587. };
  11588. @@ -30,6 +31,7 @@
  11589. #define IRQ_PER_CPU_DEVID GOT_YOU_MORON
  11590. #define IRQ_IS_POLLED GOT_YOU_MORON
  11591. #define IRQ_DISABLE_UNLAZY GOT_YOU_MORON
  11592. +#define IRQ_NO_SOFTIRQ_CALL GOT_YOU_MORON
  11593. #undef IRQF_MODIFY_MASK
  11594. #define IRQF_MODIFY_MASK GOT_YOU_MORON
  11595. @@ -40,6 +42,16 @@
  11596. desc->status_use_accessors |= (set & _IRQF_MODIFY_MASK);
  11597. }
  11598. +static inline bool irq_settings_no_softirq_call(struct irq_desc *desc)
  11599. +{
  11600. + return desc->status_use_accessors & _IRQ_NO_SOFTIRQ_CALL;
  11601. +}
  11602. +
  11603. +static inline void irq_settings_set_no_softirq_call(struct irq_desc *desc)
  11604. +{
  11605. + desc->status_use_accessors |= _IRQ_NO_SOFTIRQ_CALL;
  11606. +}
  11607. +
  11608. static inline bool irq_settings_is_per_cpu(struct irq_desc *desc)
  11609. {
  11610. return desc->status_use_accessors & _IRQ_PER_CPU;
  11611. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/irq/spurious.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/irq/spurious.c
  11612. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/irq/spurious.c 2017-04-16 10:38:29.000000000 +0200
  11613. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/irq/spurious.c 2017-04-18 17:54:26.000000000 +0200
  11614. @@ -442,6 +442,10 @@
  11615. static int __init irqfixup_setup(char *str)
  11616. {
  11617. +#ifdef CONFIG_PREEMPT_RT_BASE
  11618. + pr_warn("irqfixup boot option not supported w/ CONFIG_PREEMPT_RT_BASE\n");
  11619. + return 1;
  11620. +#endif
  11621. irqfixup = 1;
  11622. printk(KERN_WARNING "Misrouted IRQ fixup support enabled.\n");
  11623. printk(KERN_WARNING "This may impact system performance.\n");
  11624. @@ -454,6 +458,10 @@
  11625. static int __init irqpoll_setup(char *str)
  11626. {
  11627. +#ifdef CONFIG_PREEMPT_RT_BASE
  11628. + pr_warn("irqpoll boot option not supported w/ CONFIG_PREEMPT_RT_BASE\n");
  11629. + return 1;
  11630. +#endif
  11631. irqfixup = 2;
  11632. printk(KERN_WARNING "Misrouted IRQ fixup and polling support "
  11633. "enabled\n");
  11634. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/irq_work.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/irq_work.c
  11635. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/irq_work.c 2017-04-16 10:38:29.000000000 +0200
  11636. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/irq_work.c 2017-04-18 17:54:26.000000000 +0200
  11637. @@ -17,6 +17,7 @@
  11638. #include <linux/cpu.h>
  11639. #include <linux/notifier.h>
  11640. #include <linux/smp.h>
  11641. +#include <linux/interrupt.h>
  11642. #include <asm/processor.h>
  11643. @@ -65,6 +66,8 @@
  11644. */
  11645. bool irq_work_queue_on(struct irq_work *work, int cpu)
  11646. {
  11647. + struct llist_head *list;
  11648. +
  11649. /* All work should have been flushed before going offline */
  11650. WARN_ON_ONCE(cpu_is_offline(cpu));
  11651. @@ -75,7 +78,12 @@
  11652. if (!irq_work_claim(work))
  11653. return false;
  11654. - if (llist_add(&work->llnode, &per_cpu(raised_list, cpu)))
  11655. + if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL) && !(work->flags & IRQ_WORK_HARD_IRQ))
  11656. + list = &per_cpu(lazy_list, cpu);
  11657. + else
  11658. + list = &per_cpu(raised_list, cpu);
  11659. +
  11660. + if (llist_add(&work->llnode, list))
  11661. arch_send_call_function_single_ipi(cpu);
  11662. return true;
  11663. @@ -86,6 +94,9 @@
  11664. /* Enqueue the irq work @work on the current CPU */
  11665. bool irq_work_queue(struct irq_work *work)
  11666. {
  11667. + struct llist_head *list;
  11668. + bool lazy_work, realtime = IS_ENABLED(CONFIG_PREEMPT_RT_FULL);
  11669. +
  11670. /* Only queue if not already pending */
  11671. if (!irq_work_claim(work))
  11672. return false;
  11673. @@ -93,13 +104,15 @@
  11674. /* Queue the entry and raise the IPI if needed. */
  11675. preempt_disable();
  11676. - /* If the work is "lazy", handle it from next tick if any */
  11677. - if (work->flags & IRQ_WORK_LAZY) {
  11678. - if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) &&
  11679. - tick_nohz_tick_stopped())
  11680. - arch_irq_work_raise();
  11681. - } else {
  11682. - if (llist_add(&work->llnode, this_cpu_ptr(&raised_list)))
  11683. + lazy_work = work->flags & IRQ_WORK_LAZY;
  11684. +
  11685. + if (lazy_work || (realtime && !(work->flags & IRQ_WORK_HARD_IRQ)))
  11686. + list = this_cpu_ptr(&lazy_list);
  11687. + else
  11688. + list = this_cpu_ptr(&raised_list);
  11689. +
  11690. + if (llist_add(&work->llnode, list)) {
  11691. + if (!lazy_work || tick_nohz_tick_stopped())
  11692. arch_irq_work_raise();
  11693. }
  11694. @@ -116,9 +129,8 @@
  11695. raised = this_cpu_ptr(&raised_list);
  11696. lazy = this_cpu_ptr(&lazy_list);
  11697. - if (llist_empty(raised) || arch_irq_work_has_interrupt())
  11698. - if (llist_empty(lazy))
  11699. - return false;
  11700. + if (llist_empty(raised) && llist_empty(lazy))
  11701. + return false;
  11702. /* All work should have been flushed before going offline */
  11703. WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
  11704. @@ -132,7 +144,7 @@
  11705. struct irq_work *work;
  11706. struct llist_node *llnode;
  11707. - BUG_ON(!irqs_disabled());
  11708. + BUG_ON_NONRT(!irqs_disabled());
  11709. if (llist_empty(list))
  11710. return;
  11711. @@ -169,7 +181,16 @@
  11712. void irq_work_run(void)
  11713. {
  11714. irq_work_run_list(this_cpu_ptr(&raised_list));
  11715. - irq_work_run_list(this_cpu_ptr(&lazy_list));
  11716. + if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL)) {
  11717. + /*
  11718. + * NOTE: we raise softirq via IPI for safety,
  11719. + * and execute in irq_work_tick() to move the
  11720. + * overhead from hard to soft irq context.
  11721. + */
  11722. + if (!llist_empty(this_cpu_ptr(&lazy_list)))
  11723. + raise_softirq(TIMER_SOFTIRQ);
  11724. + } else
  11725. + irq_work_run_list(this_cpu_ptr(&lazy_list));
  11726. }
  11727. EXPORT_SYMBOL_GPL(irq_work_run);
  11728. @@ -179,8 +200,17 @@
  11729. if (!llist_empty(raised) && !arch_irq_work_has_interrupt())
  11730. irq_work_run_list(raised);
  11731. +
  11732. + if (!IS_ENABLED(CONFIG_PREEMPT_RT_FULL))
  11733. + irq_work_run_list(this_cpu_ptr(&lazy_list));
  11734. +}
  11735. +
  11736. +#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_PREEMPT_RT_FULL)
  11737. +void irq_work_tick_soft(void)
  11738. +{
  11739. irq_work_run_list(this_cpu_ptr(&lazy_list));
  11740. }
  11741. +#endif
  11742. /*
  11743. * Synchronize against the irq_work @entry, ensures the entry is not
  11744. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/ksysfs.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/ksysfs.c
  11745. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/ksysfs.c 2017-04-16 10:38:29.000000000 +0200
  11746. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/ksysfs.c 2017-04-18 17:54:26.000000000 +0200
  11747. @@ -136,6 +136,15 @@
  11748. #endif /* CONFIG_KEXEC_CORE */
  11749. +#if defined(CONFIG_PREEMPT_RT_FULL)
  11750. +static ssize_t realtime_show(struct kobject *kobj,
  11751. + struct kobj_attribute *attr, char *buf)
  11752. +{
  11753. + return sprintf(buf, "%d\n", 1);
  11754. +}
  11755. +KERNEL_ATTR_RO(realtime);
  11756. +#endif
  11757. +
  11758. /* whether file capabilities are enabled */
  11759. static ssize_t fscaps_show(struct kobject *kobj,
  11760. struct kobj_attribute *attr, char *buf)
  11761. @@ -225,6 +234,9 @@
  11762. &rcu_expedited_attr.attr,
  11763. &rcu_normal_attr.attr,
  11764. #endif
  11765. +#ifdef CONFIG_PREEMPT_RT_FULL
  11766. + &realtime_attr.attr,
  11767. +#endif
  11768. NULL
  11769. };
  11770. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/Makefile linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/Makefile
  11771. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/Makefile 2017-04-16 10:38:29.000000000 +0200
  11772. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/Makefile 2017-04-18 17:54:26.000000000 +0200
  11773. @@ -2,7 +2,7 @@
  11774. # and is generally not a function of system call inputs.
  11775. KCOV_INSTRUMENT := n
  11776. -obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
  11777. +obj-y += semaphore.o percpu-rwsem.o
  11778. ifdef CONFIG_FUNCTION_TRACER
  11779. CFLAGS_REMOVE_lockdep.o = $(CC_FLAGS_FTRACE)
  11780. @@ -11,7 +11,11 @@
  11781. CFLAGS_REMOVE_rtmutex-debug.o = $(CC_FLAGS_FTRACE)
  11782. endif
  11783. +ifneq ($(CONFIG_PREEMPT_RT_FULL),y)
  11784. +obj-y += mutex.o
  11785. obj-$(CONFIG_DEBUG_MUTEXES) += mutex-debug.o
  11786. +endif
  11787. +obj-y += rwsem.o
  11788. obj-$(CONFIG_LOCKDEP) += lockdep.o
  11789. ifeq ($(CONFIG_PROC_FS),y)
  11790. obj-$(CONFIG_LOCKDEP) += lockdep_proc.o
  11791. @@ -24,7 +28,10 @@
  11792. obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
  11793. obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o
  11794. obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o
  11795. +ifneq ($(CONFIG_PREEMPT_RT_FULL),y)
  11796. obj-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o
  11797. obj-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem-xadd.o
  11798. +endif
  11799. +obj-$(CONFIG_PREEMPT_RT_FULL) += rt.o rwsem-rt.o
  11800. obj-$(CONFIG_QUEUED_RWLOCKS) += qrwlock.o
  11801. obj-$(CONFIG_LOCK_TORTURE_TEST) += locktorture.o
  11802. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/lockdep.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/lockdep.c
  11803. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/lockdep.c 2017-04-16 10:38:29.000000000 +0200
  11804. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/lockdep.c 2017-04-18 17:54:26.000000000 +0200
  11805. @@ -658,6 +658,7 @@
  11806. struct lockdep_subclass_key *key;
  11807. struct hlist_head *hash_head;
  11808. struct lock_class *class;
  11809. + bool is_static = false;
  11810. if (unlikely(subclass >= MAX_LOCKDEP_SUBCLASSES)) {
  11811. debug_locks_off();
  11812. @@ -671,10 +672,23 @@
  11813. /*
  11814. * Static locks do not have their class-keys yet - for them the key
  11815. - * is the lock object itself:
  11816. - */
  11817. - if (unlikely(!lock->key))
  11818. - lock->key = (void *)lock;
  11819. + * is the lock object itself. If the lock is in the per cpu area,
  11820. + * the canonical address of the lock (per cpu offset removed) is
  11821. + * used.
  11822. + */
  11823. + if (unlikely(!lock->key)) {
  11824. + unsigned long can_addr, addr = (unsigned long)lock;
  11825. +
  11826. + if (__is_kernel_percpu_address(addr, &can_addr))
  11827. + lock->key = (void *)can_addr;
  11828. + else if (__is_module_percpu_address(addr, &can_addr))
  11829. + lock->key = (void *)can_addr;
  11830. + else if (static_obj(lock))
  11831. + lock->key = (void *)lock;
  11832. + else
  11833. + return ERR_PTR(-EINVAL);
  11834. + is_static = true;
  11835. + }
  11836. /*
  11837. * NOTE: the class-key must be unique. For dynamic locks, a static
  11838. @@ -706,7 +720,7 @@
  11839. }
  11840. }
  11841. - return NULL;
  11842. + return is_static || static_obj(lock->key) ? NULL : ERR_PTR(-EINVAL);
  11843. }
  11844. /*
  11845. @@ -724,19 +738,18 @@
  11846. DEBUG_LOCKS_WARN_ON(!irqs_disabled());
  11847. class = look_up_lock_class(lock, subclass);
  11848. - if (likely(class))
  11849. + if (likely(!IS_ERR_OR_NULL(class)))
  11850. goto out_set_class_cache;
  11851. /*
  11852. * Debug-check: all keys must be persistent!
  11853. - */
  11854. - if (!static_obj(lock->key)) {
  11855. + */
  11856. + if (IS_ERR(class)) {
  11857. debug_locks_off();
  11858. printk("INFO: trying to register non-static key.\n");
  11859. printk("the code is fine but needs lockdep annotation.\n");
  11860. printk("turning off the locking correctness validator.\n");
  11861. dump_stack();
  11862. -
  11863. return NULL;
  11864. }
  11865. @@ -3410,7 +3423,7 @@
  11866. * Clearly if the lock hasn't been acquired _ever_, we're not
  11867. * holding it either, so report failure.
  11868. */
  11869. - if (!class)
  11870. + if (IS_ERR_OR_NULL(class))
  11871. return 0;
  11872. /*
  11873. @@ -3689,6 +3702,7 @@
  11874. }
  11875. }
  11876. +#ifndef CONFIG_PREEMPT_RT_FULL
  11877. /*
  11878. * We dont accurately track softirq state in e.g.
  11879. * hardirq contexts (such as on 4KSTACKS), so only
  11880. @@ -3703,6 +3717,7 @@
  11881. DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled);
  11882. }
  11883. }
  11884. +#endif
  11885. if (!debug_locks)
  11886. print_irqtrace_events(current);
  11887. @@ -4159,7 +4174,7 @@
  11888. * If the class exists we look it up and zap it:
  11889. */
  11890. class = look_up_lock_class(lock, j);
  11891. - if (class)
  11892. + if (!IS_ERR_OR_NULL(class))
  11893. zap_class(class);
  11894. }
  11895. /*
  11896. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/locktorture.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/locktorture.c
  11897. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/locktorture.c 2017-04-16 10:38:29.000000000 +0200
  11898. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/locktorture.c 2017-04-18 17:54:26.000000000 +0200
  11899. @@ -26,7 +26,6 @@
  11900. #include <linux/kthread.h>
  11901. #include <linux/sched/rt.h>
  11902. #include <linux/spinlock.h>
  11903. -#include <linux/rwlock.h>
  11904. #include <linux/mutex.h>
  11905. #include <linux/rwsem.h>
  11906. #include <linux/smp.h>
  11907. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/percpu-rwsem.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/percpu-rwsem.c
  11908. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/percpu-rwsem.c 2017-04-16 10:38:29.000000000 +0200
  11909. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/percpu-rwsem.c 2017-04-18 17:54:26.000000000 +0200
  11910. @@ -18,7 +18,7 @@
  11911. /* ->rw_sem represents the whole percpu_rw_semaphore for lockdep */
  11912. rcu_sync_init(&sem->rss, RCU_SCHED_SYNC);
  11913. __init_rwsem(&sem->rw_sem, name, rwsem_key);
  11914. - init_waitqueue_head(&sem->writer);
  11915. + init_swait_queue_head(&sem->writer);
  11916. sem->readers_block = 0;
  11917. return 0;
  11918. }
  11919. @@ -103,7 +103,7 @@
  11920. __this_cpu_dec(*sem->read_count);
  11921. /* Prod writer to recheck readers_active */
  11922. - wake_up(&sem->writer);
  11923. + swake_up(&sem->writer);
  11924. }
  11925. EXPORT_SYMBOL_GPL(__percpu_up_read);
  11926. @@ -160,7 +160,7 @@
  11927. */
  11928. /* Wait for all now active readers to complete. */
  11929. - wait_event(sem->writer, readers_active_check(sem));
  11930. + swait_event(sem->writer, readers_active_check(sem));
  11931. }
  11932. EXPORT_SYMBOL_GPL(percpu_down_write);
  11933. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rt.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rt.c
  11934. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rt.c 1970-01-01 01:00:00.000000000 +0100
  11935. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rt.c 2017-04-18 17:54:26.000000000 +0200
  11936. @@ -0,0 +1,331 @@
  11937. +/*
  11938. + * kernel/rt.c
  11939. + *
  11940. + * Real-Time Preemption Support
  11941. + *
  11942. + * started by Ingo Molnar:
  11943. + *
  11944. + * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  11945. + * Copyright (C) 2006, Timesys Corp., Thomas Gleixner <tglx@timesys.com>
  11946. + *
  11947. + * historic credit for proving that Linux spinlocks can be implemented via
  11948. + * RT-aware mutexes goes to many people: The Pmutex project (Dirk Grambow
  11949. + * and others) who prototyped it on 2.4 and did lots of comparative
  11950. + * research and analysis; TimeSys, for proving that you can implement a
  11951. + * fully preemptible kernel via the use of IRQ threading and mutexes;
  11952. + * Bill Huey for persuasively arguing on lkml that the mutex model is the
  11953. + * right one; and to MontaVista, who ported pmutexes to 2.6.
  11954. + *
  11955. + * This code is a from-scratch implementation and is not based on pmutexes,
  11956. + * but the idea of converting spinlocks to mutexes is used here too.
  11957. + *
  11958. + * lock debugging, locking tree, deadlock detection:
  11959. + *
  11960. + * Copyright (C) 2004, LynuxWorks, Inc., Igor Manyilov, Bill Huey
  11961. + * Released under the General Public License (GPL).
  11962. + *
  11963. + * Includes portions of the generic R/W semaphore implementation from:
  11964. + *
  11965. + * Copyright (c) 2001 David Howells (dhowells@redhat.com).
  11966. + * - Derived partially from idea by Andrea Arcangeli <andrea@suse.de>
  11967. + * - Derived also from comments by Linus
  11968. + *
  11969. + * Pending ownership of locks and ownership stealing:
  11970. + *
  11971. + * Copyright (C) 2005, Kihon Technologies Inc., Steven Rostedt
  11972. + *
  11973. + * (also by Steven Rostedt)
  11974. + * - Converted single pi_lock to individual task locks.
  11975. + *
  11976. + * By Esben Nielsen:
  11977. + * Doing priority inheritance with help of the scheduler.
  11978. + *
  11979. + * Copyright (C) 2006, Timesys Corp., Thomas Gleixner <tglx@timesys.com>
  11980. + * - major rework based on Esben Nielsens initial patch
  11981. + * - replaced thread_info references by task_struct refs
  11982. + * - removed task->pending_owner dependency
  11983. + * - BKL drop/reacquire for semaphore style locks to avoid deadlocks
  11984. + * in the scheduler return path as discussed with Steven Rostedt
  11985. + *
  11986. + * Copyright (C) 2006, Kihon Technologies Inc.
  11987. + * Steven Rostedt <rostedt@goodmis.org>
  11988. + * - debugged and patched Thomas Gleixner's rework.
  11989. + * - added back the cmpxchg to the rework.
  11990. + * - turned atomic require back on for SMP.
  11991. + */
  11992. +
  11993. +#include <linux/spinlock.h>
  11994. +#include <linux/rtmutex.h>
  11995. +#include <linux/sched.h>
  11996. +#include <linux/delay.h>
  11997. +#include <linux/module.h>
  11998. +#include <linux/kallsyms.h>
  11999. +#include <linux/syscalls.h>
  12000. +#include <linux/interrupt.h>
  12001. +#include <linux/plist.h>
  12002. +#include <linux/fs.h>
  12003. +#include <linux/futex.h>
  12004. +#include <linux/hrtimer.h>
  12005. +
  12006. +#include "rtmutex_common.h"
  12007. +
  12008. +/*
  12009. + * struct mutex functions
  12010. + */
  12011. +void __mutex_do_init(struct mutex *mutex, const char *name,
  12012. + struct lock_class_key *key)
  12013. +{
  12014. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  12015. + /*
  12016. + * Make sure we are not reinitializing a held lock:
  12017. + */
  12018. + debug_check_no_locks_freed((void *)mutex, sizeof(*mutex));
  12019. + lockdep_init_map(&mutex->dep_map, name, key, 0);
  12020. +#endif
  12021. + mutex->lock.save_state = 0;
  12022. +}
  12023. +EXPORT_SYMBOL(__mutex_do_init);
  12024. +
  12025. +void __lockfunc _mutex_lock(struct mutex *lock)
  12026. +{
  12027. + mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  12028. + rt_mutex_lock(&lock->lock);
  12029. +}
  12030. +EXPORT_SYMBOL(_mutex_lock);
  12031. +
  12032. +int __lockfunc _mutex_lock_interruptible(struct mutex *lock)
  12033. +{
  12034. + int ret;
  12035. +
  12036. + mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  12037. + ret = rt_mutex_lock_interruptible(&lock->lock);
  12038. + if (ret)
  12039. + mutex_release(&lock->dep_map, 1, _RET_IP_);
  12040. + return ret;
  12041. +}
  12042. +EXPORT_SYMBOL(_mutex_lock_interruptible);
  12043. +
  12044. +int __lockfunc _mutex_lock_killable(struct mutex *lock)
  12045. +{
  12046. + int ret;
  12047. +
  12048. + mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  12049. + ret = rt_mutex_lock_killable(&lock->lock);
  12050. + if (ret)
  12051. + mutex_release(&lock->dep_map, 1, _RET_IP_);
  12052. + return ret;
  12053. +}
  12054. +EXPORT_SYMBOL(_mutex_lock_killable);
  12055. +
  12056. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  12057. +void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass)
  12058. +{
  12059. + mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_);
  12060. + rt_mutex_lock(&lock->lock);
  12061. +}
  12062. +EXPORT_SYMBOL(_mutex_lock_nested);
  12063. +
  12064. +void __lockfunc _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
  12065. +{
  12066. + mutex_acquire_nest(&lock->dep_map, 0, 0, nest, _RET_IP_);
  12067. + rt_mutex_lock(&lock->lock);
  12068. +}
  12069. +EXPORT_SYMBOL(_mutex_lock_nest_lock);
  12070. +
  12071. +int __lockfunc _mutex_lock_interruptible_nested(struct mutex *lock, int subclass)
  12072. +{
  12073. + int ret;
  12074. +
  12075. + mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_);
  12076. + ret = rt_mutex_lock_interruptible(&lock->lock);
  12077. + if (ret)
  12078. + mutex_release(&lock->dep_map, 1, _RET_IP_);
  12079. + return ret;
  12080. +}
  12081. +EXPORT_SYMBOL(_mutex_lock_interruptible_nested);
  12082. +
  12083. +int __lockfunc _mutex_lock_killable_nested(struct mutex *lock, int subclass)
  12084. +{
  12085. + int ret;
  12086. +
  12087. + mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
  12088. + ret = rt_mutex_lock_killable(&lock->lock);
  12089. + if (ret)
  12090. + mutex_release(&lock->dep_map, 1, _RET_IP_);
  12091. + return ret;
  12092. +}
  12093. +EXPORT_SYMBOL(_mutex_lock_killable_nested);
  12094. +#endif
  12095. +
  12096. +int __lockfunc _mutex_trylock(struct mutex *lock)
  12097. +{
  12098. + int ret = rt_mutex_trylock(&lock->lock);
  12099. +
  12100. + if (ret)
  12101. + mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  12102. +
  12103. + return ret;
  12104. +}
  12105. +EXPORT_SYMBOL(_mutex_trylock);
  12106. +
  12107. +void __lockfunc _mutex_unlock(struct mutex *lock)
  12108. +{
  12109. + mutex_release(&lock->dep_map, 1, _RET_IP_);
  12110. + rt_mutex_unlock(&lock->lock);
  12111. +}
  12112. +EXPORT_SYMBOL(_mutex_unlock);
  12113. +
  12114. +/*
  12115. + * rwlock_t functions
  12116. + */
  12117. +int __lockfunc rt_write_trylock(rwlock_t *rwlock)
  12118. +{
  12119. + int ret;
  12120. +
  12121. + migrate_disable();
  12122. + ret = rt_mutex_trylock(&rwlock->lock);
  12123. + if (ret)
  12124. + rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_);
  12125. + else
  12126. + migrate_enable();
  12127. +
  12128. + return ret;
  12129. +}
  12130. +EXPORT_SYMBOL(rt_write_trylock);
  12131. +
  12132. +int __lockfunc rt_write_trylock_irqsave(rwlock_t *rwlock, unsigned long *flags)
  12133. +{
  12134. + int ret;
  12135. +
  12136. + *flags = 0;
  12137. + ret = rt_write_trylock(rwlock);
  12138. + return ret;
  12139. +}
  12140. +EXPORT_SYMBOL(rt_write_trylock_irqsave);
  12141. +
  12142. +int __lockfunc rt_read_trylock(rwlock_t *rwlock)
  12143. +{
  12144. + struct rt_mutex *lock = &rwlock->lock;
  12145. + int ret = 1;
  12146. +
  12147. + /*
  12148. + * recursive read locks succeed when current owns the lock,
  12149. + * but not when read_depth == 0 which means that the lock is
  12150. + * write locked.
  12151. + */
  12152. + if (rt_mutex_owner(lock) != current) {
  12153. + migrate_disable();
  12154. + ret = rt_mutex_trylock(lock);
  12155. + if (ret)
  12156. + rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_);
  12157. + else
  12158. + migrate_enable();
  12159. +
  12160. + } else if (!rwlock->read_depth) {
  12161. + ret = 0;
  12162. + }
  12163. +
  12164. + if (ret)
  12165. + rwlock->read_depth++;
  12166. +
  12167. + return ret;
  12168. +}
  12169. +EXPORT_SYMBOL(rt_read_trylock);
  12170. +
  12171. +void __lockfunc rt_write_lock(rwlock_t *rwlock)
  12172. +{
  12173. + rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_);
  12174. + __rt_spin_lock(&rwlock->lock);
  12175. +}
  12176. +EXPORT_SYMBOL(rt_write_lock);
  12177. +
  12178. +void __lockfunc rt_read_lock(rwlock_t *rwlock)
  12179. +{
  12180. + struct rt_mutex *lock = &rwlock->lock;
  12181. +
  12182. +
  12183. + /*
  12184. + * recursive read locks succeed when current owns the lock
  12185. + */
  12186. + if (rt_mutex_owner(lock) != current) {
  12187. + rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_);
  12188. + __rt_spin_lock(lock);
  12189. + }
  12190. + rwlock->read_depth++;
  12191. +}
  12192. +
  12193. +EXPORT_SYMBOL(rt_read_lock);
  12194. +
  12195. +void __lockfunc rt_write_unlock(rwlock_t *rwlock)
  12196. +{
  12197. + /* NOTE: we always pass in '1' for nested, for simplicity */
  12198. + rwlock_release(&rwlock->dep_map, 1, _RET_IP_);
  12199. + __rt_spin_unlock(&rwlock->lock);
  12200. + migrate_enable();
  12201. +}
  12202. +EXPORT_SYMBOL(rt_write_unlock);
  12203. +
  12204. +void __lockfunc rt_read_unlock(rwlock_t *rwlock)
  12205. +{
  12206. + /* Release the lock only when read_depth is down to 0 */
  12207. + if (--rwlock->read_depth == 0) {
  12208. + rwlock_release(&rwlock->dep_map, 1, _RET_IP_);
  12209. + __rt_spin_unlock(&rwlock->lock);
  12210. + migrate_enable();
  12211. + }
  12212. +}
  12213. +EXPORT_SYMBOL(rt_read_unlock);
  12214. +
  12215. +unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock)
  12216. +{
  12217. + rt_write_lock(rwlock);
  12218. +
  12219. + return 0;
  12220. +}
  12221. +EXPORT_SYMBOL(rt_write_lock_irqsave);
  12222. +
  12223. +unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock)
  12224. +{
  12225. + rt_read_lock(rwlock);
  12226. +
  12227. + return 0;
  12228. +}
  12229. +EXPORT_SYMBOL(rt_read_lock_irqsave);
  12230. +
  12231. +void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key)
  12232. +{
  12233. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  12234. + /*
  12235. + * Make sure we are not reinitializing a held lock:
  12236. + */
  12237. + debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock));
  12238. + lockdep_init_map(&rwlock->dep_map, name, key, 0);
  12239. +#endif
  12240. + rwlock->lock.save_state = 1;
  12241. + rwlock->read_depth = 0;
  12242. +}
  12243. +EXPORT_SYMBOL(__rt_rwlock_init);
  12244. +
  12245. +/**
  12246. + * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
  12247. + * @cnt: the atomic which we are to dec
  12248. + * @lock: the mutex to return holding if we dec to 0
  12249. + *
  12250. + * return true and hold lock if we dec to 0, return false otherwise
  12251. + */
  12252. +int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
  12253. +{
  12254. + /* dec if we can't possibly hit 0 */
  12255. + if (atomic_add_unless(cnt, -1, 1))
  12256. + return 0;
  12257. + /* we might hit 0, so take the lock */
  12258. + mutex_lock(lock);
  12259. + if (!atomic_dec_and_test(cnt)) {
  12260. + /* when we actually did the dec, we didn't hit 0 */
  12261. + mutex_unlock(lock);
  12262. + return 0;
  12263. + }
  12264. + /* we hit 0, and we hold the lock */
  12265. + return 1;
  12266. +}
  12267. +EXPORT_SYMBOL(atomic_dec_and_mutex_lock);
  12268. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rtmutex-debug.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rtmutex-debug.c
  12269. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rtmutex-debug.c 2017-04-16 10:38:29.000000000 +0200
  12270. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rtmutex-debug.c 2017-04-18 17:54:26.000000000 +0200
  12271. @@ -173,12 +173,3 @@
  12272. lock->name = name;
  12273. }
  12274. -void
  12275. -rt_mutex_deadlock_account_lock(struct rt_mutex *lock, struct task_struct *task)
  12276. -{
  12277. -}
  12278. -
  12279. -void rt_mutex_deadlock_account_unlock(struct task_struct *task)
  12280. -{
  12281. -}
  12282. -
  12283. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rtmutex-debug.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rtmutex-debug.h
  12284. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rtmutex-debug.h 2017-04-16 10:38:29.000000000 +0200
  12285. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rtmutex-debug.h 2017-04-18 17:54:26.000000000 +0200
  12286. @@ -9,9 +9,6 @@
  12287. * This file contains macros used solely by rtmutex.c. Debug version.
  12288. */
  12289. -extern void
  12290. -rt_mutex_deadlock_account_lock(struct rt_mutex *lock, struct task_struct *task);
  12291. -extern void rt_mutex_deadlock_account_unlock(struct task_struct *task);
  12292. extern void debug_rt_mutex_init_waiter(struct rt_mutex_waiter *waiter);
  12293. extern void debug_rt_mutex_free_waiter(struct rt_mutex_waiter *waiter);
  12294. extern void debug_rt_mutex_init(struct rt_mutex *lock, const char *name);
  12295. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rtmutex.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rtmutex.c
  12296. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rtmutex.c 2017-04-16 10:38:29.000000000 +0200
  12297. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rtmutex.c 2017-04-18 17:54:26.000000000 +0200
  12298. @@ -7,6 +7,11 @@
  12299. * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
  12300. * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
  12301. * Copyright (C) 2006 Esben Nielsen
  12302. + * Adaptive Spinlocks:
  12303. + * Copyright (C) 2008 Novell, Inc., Gregory Haskins, Sven Dietrich,
  12304. + * and Peter Morreale,
  12305. + * Adaptive Spinlocks simplification:
  12306. + * Copyright (C) 2008 Red Hat, Inc., Steven Rostedt <srostedt@redhat.com>
  12307. *
  12308. * See Documentation/locking/rt-mutex-design.txt for details.
  12309. */
  12310. @@ -16,6 +21,7 @@
  12311. #include <linux/sched/rt.h>
  12312. #include <linux/sched/deadline.h>
  12313. #include <linux/timer.h>
  12314. +#include <linux/ww_mutex.h>
  12315. #include "rtmutex_common.h"
  12316. @@ -133,6 +139,12 @@
  12317. WRITE_ONCE(*p, owner & ~RT_MUTEX_HAS_WAITERS);
  12318. }
  12319. +static int rt_mutex_real_waiter(struct rt_mutex_waiter *waiter)
  12320. +{
  12321. + return waiter && waiter != PI_WAKEUP_INPROGRESS &&
  12322. + waiter != PI_REQUEUE_INPROGRESS;
  12323. +}
  12324. +
  12325. /*
  12326. * We can speed up the acquire/release, if there's no debugging state to be
  12327. * set up.
  12328. @@ -414,6 +426,14 @@
  12329. return debug_rt_mutex_detect_deadlock(waiter, chwalk);
  12330. }
  12331. +static void rt_mutex_wake_waiter(struct rt_mutex_waiter *waiter)
  12332. +{
  12333. + if (waiter->savestate)
  12334. + wake_up_lock_sleeper(waiter->task);
  12335. + else
  12336. + wake_up_process(waiter->task);
  12337. +}
  12338. +
  12339. /*
  12340. * Max number of times we'll walk the boosting chain:
  12341. */
  12342. @@ -421,7 +441,8 @@
  12343. static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
  12344. {
  12345. - return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;
  12346. + return rt_mutex_real_waiter(p->pi_blocked_on) ?
  12347. + p->pi_blocked_on->lock : NULL;
  12348. }
  12349. /*
  12350. @@ -557,7 +578,7 @@
  12351. * reached or the state of the chain has changed while we
  12352. * dropped the locks.
  12353. */
  12354. - if (!waiter)
  12355. + if (!rt_mutex_real_waiter(waiter))
  12356. goto out_unlock_pi;
  12357. /*
  12358. @@ -719,13 +740,16 @@
  12359. * follow here. This is the end of the chain we are walking.
  12360. */
  12361. if (!rt_mutex_owner(lock)) {
  12362. + struct rt_mutex_waiter *lock_top_waiter;
  12363. +
  12364. /*
  12365. * If the requeue [7] above changed the top waiter,
  12366. * then we need to wake the new top waiter up to try
  12367. * to get the lock.
  12368. */
  12369. - if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))
  12370. - wake_up_process(rt_mutex_top_waiter(lock)->task);
  12371. + lock_top_waiter = rt_mutex_top_waiter(lock);
  12372. + if (prerequeue_top_waiter != lock_top_waiter)
  12373. + rt_mutex_wake_waiter(lock_top_waiter);
  12374. raw_spin_unlock_irq(&lock->wait_lock);
  12375. return 0;
  12376. }
  12377. @@ -818,6 +842,25 @@
  12378. return ret;
  12379. }
  12380. +
  12381. +#define STEAL_NORMAL 0
  12382. +#define STEAL_LATERAL 1
  12383. +
  12384. +/*
  12385. + * Note that RT tasks are excluded from lateral-steals to prevent the
  12386. + * introduction of an unbounded latency
  12387. + */
  12388. +static inline int lock_is_stealable(struct task_struct *task,
  12389. + struct task_struct *pendowner, int mode)
  12390. +{
  12391. + if (mode == STEAL_NORMAL || rt_task(task)) {
  12392. + if (task->prio >= pendowner->prio)
  12393. + return 0;
  12394. + } else if (task->prio > pendowner->prio)
  12395. + return 0;
  12396. + return 1;
  12397. +}
  12398. +
  12399. /*
  12400. * Try to take an rt-mutex
  12401. *
  12402. @@ -828,8 +871,9 @@
  12403. * @waiter: The waiter that is queued to the lock's wait tree if the
  12404. * callsite called task_blocked_on_lock(), otherwise NULL
  12405. */
  12406. -static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
  12407. - struct rt_mutex_waiter *waiter)
  12408. +static int __try_to_take_rt_mutex(struct rt_mutex *lock,
  12409. + struct task_struct *task,
  12410. + struct rt_mutex_waiter *waiter, int mode)
  12411. {
  12412. /*
  12413. * Before testing whether we can acquire @lock, we set the
  12414. @@ -866,8 +910,10 @@
  12415. * If waiter is not the highest priority waiter of
  12416. * @lock, give up.
  12417. */
  12418. - if (waiter != rt_mutex_top_waiter(lock))
  12419. + if (waiter != rt_mutex_top_waiter(lock)) {
  12420. + /* XXX lock_is_stealable() ? */
  12421. return 0;
  12422. + }
  12423. /*
  12424. * We can acquire the lock. Remove the waiter from the
  12425. @@ -885,14 +931,10 @@
  12426. * not need to be dequeued.
  12427. */
  12428. if (rt_mutex_has_waiters(lock)) {
  12429. - /*
  12430. - * If @task->prio is greater than or equal to
  12431. - * the top waiter priority (kernel view),
  12432. - * @task lost.
  12433. - */
  12434. - if (task->prio >= rt_mutex_top_waiter(lock)->prio)
  12435. - return 0;
  12436. + struct task_struct *pown = rt_mutex_top_waiter(lock)->task;
  12437. + if (task != pown && !lock_is_stealable(task, pown, mode))
  12438. + return 0;
  12439. /*
  12440. * The current top waiter stays enqueued. We
  12441. * don't have to change anything in the lock
  12442. @@ -936,10 +978,394 @@
  12443. */
  12444. rt_mutex_set_owner(lock, task);
  12445. - rt_mutex_deadlock_account_lock(lock, task);
  12446. + return 1;
  12447. +}
  12448. +
  12449. +#ifdef CONFIG_PREEMPT_RT_FULL
  12450. +/*
  12451. + * preemptible spin_lock functions:
  12452. + */
  12453. +static inline void rt_spin_lock_fastlock(struct rt_mutex *lock,
  12454. + void (*slowfn)(struct rt_mutex *lock,
  12455. + bool mg_off),
  12456. + bool do_mig_dis)
  12457. +{
  12458. + might_sleep_no_state_check();
  12459. +
  12460. + if (do_mig_dis)
  12461. + migrate_disable();
  12462. +
  12463. + if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
  12464. + return;
  12465. + else
  12466. + slowfn(lock, do_mig_dis);
  12467. +}
  12468. +
  12469. +static inline void rt_spin_lock_fastunlock(struct rt_mutex *lock,
  12470. + void (*slowfn)(struct rt_mutex *lock))
  12471. +{
  12472. + if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))
  12473. + return;
  12474. + else
  12475. + slowfn(lock);
  12476. +}
  12477. +#ifdef CONFIG_SMP
  12478. +/*
  12479. + * Note that owner is a speculative pointer and dereferencing relies
  12480. + * on rcu_read_lock() and the check against the lock owner.
  12481. + */
  12482. +static int adaptive_wait(struct rt_mutex *lock,
  12483. + struct task_struct *owner)
  12484. +{
  12485. + int res = 0;
  12486. + rcu_read_lock();
  12487. + for (;;) {
  12488. + if (owner != rt_mutex_owner(lock))
  12489. + break;
  12490. + /*
  12491. + * Ensure that owner->on_cpu is dereferenced _after_
  12492. + * checking the above to be valid.
  12493. + */
  12494. + barrier();
  12495. + if (!owner->on_cpu) {
  12496. + res = 1;
  12497. + break;
  12498. + }
  12499. + cpu_relax();
  12500. + }
  12501. + rcu_read_unlock();
  12502. + return res;
  12503. +}
  12504. +#else
  12505. +static int adaptive_wait(struct rt_mutex *lock,
  12506. + struct task_struct *orig_owner)
  12507. +{
  12508. return 1;
  12509. }
  12510. +#endif
  12511. +
  12512. +static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
  12513. + struct rt_mutex_waiter *waiter,
  12514. + struct task_struct *task,
  12515. + enum rtmutex_chainwalk chwalk);
  12516. +/*
  12517. + * Slow path lock function spin_lock style: this variant is very
  12518. + * careful not to miss any non-lock wakeups.
  12519. + *
  12520. + * We store the current state under p->pi_lock in p->saved_state and
  12521. + * the try_to_wake_up() code handles this accordingly.
  12522. + */
  12523. +static void noinline __sched rt_spin_lock_slowlock(struct rt_mutex *lock,
  12524. + bool mg_off)
  12525. +{
  12526. + struct task_struct *lock_owner, *self = current;
  12527. + struct rt_mutex_waiter waiter, *top_waiter;
  12528. + unsigned long flags;
  12529. + int ret;
  12530. +
  12531. + rt_mutex_init_waiter(&waiter, true);
  12532. +
  12533. + raw_spin_lock_irqsave(&lock->wait_lock, flags);
  12534. +
  12535. + if (__try_to_take_rt_mutex(lock, self, NULL, STEAL_LATERAL)) {
  12536. + raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  12537. + return;
  12538. + }
  12539. +
  12540. + BUG_ON(rt_mutex_owner(lock) == self);
  12541. +
  12542. + /*
  12543. + * We save whatever state the task is in and we'll restore it
  12544. + * after acquiring the lock taking real wakeups into account
  12545. + * as well. We are serialized via pi_lock against wakeups. See
  12546. + * try_to_wake_up().
  12547. + */
  12548. + raw_spin_lock(&self->pi_lock);
  12549. + self->saved_state = self->state;
  12550. + __set_current_state_no_track(TASK_UNINTERRUPTIBLE);
  12551. + raw_spin_unlock(&self->pi_lock);
  12552. +
  12553. + ret = task_blocks_on_rt_mutex(lock, &waiter, self, RT_MUTEX_MIN_CHAINWALK);
  12554. + BUG_ON(ret);
  12555. +
  12556. + for (;;) {
  12557. + /* Try to acquire the lock again. */
  12558. + if (__try_to_take_rt_mutex(lock, self, &waiter, STEAL_LATERAL))
  12559. + break;
  12560. +
  12561. + top_waiter = rt_mutex_top_waiter(lock);
  12562. + lock_owner = rt_mutex_owner(lock);
  12563. +
  12564. + raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  12565. +
  12566. + debug_rt_mutex_print_deadlock(&waiter);
  12567. +
  12568. + if (top_waiter != &waiter || adaptive_wait(lock, lock_owner)) {
  12569. + if (mg_off)
  12570. + migrate_enable();
  12571. + schedule();
  12572. + if (mg_off)
  12573. + migrate_disable();
  12574. + }
  12575. +
  12576. + raw_spin_lock_irqsave(&lock->wait_lock, flags);
  12577. +
  12578. + raw_spin_lock(&self->pi_lock);
  12579. + __set_current_state_no_track(TASK_UNINTERRUPTIBLE);
  12580. + raw_spin_unlock(&self->pi_lock);
  12581. + }
  12582. +
  12583. + /*
  12584. + * Restore the task state to current->saved_state. We set it
  12585. + * to the original state above and the try_to_wake_up() code
  12586. + * has possibly updated it when a real (non-rtmutex) wakeup
  12587. + * happened while we were blocked. Clear saved_state so
  12588. + * try_to_wakeup() does not get confused.
  12589. + */
  12590. + raw_spin_lock(&self->pi_lock);
  12591. + __set_current_state_no_track(self->saved_state);
  12592. + self->saved_state = TASK_RUNNING;
  12593. + raw_spin_unlock(&self->pi_lock);
  12594. +
  12595. + /*
  12596. + * try_to_take_rt_mutex() sets the waiter bit
  12597. + * unconditionally. We might have to fix that up:
  12598. + */
  12599. + fixup_rt_mutex_waiters(lock);
  12600. +
  12601. + BUG_ON(rt_mutex_has_waiters(lock) && &waiter == rt_mutex_top_waiter(lock));
  12602. + BUG_ON(!RB_EMPTY_NODE(&waiter.tree_entry));
  12603. +
  12604. + raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  12605. +
  12606. + debug_rt_mutex_free_waiter(&waiter);
  12607. +}
  12608. +
  12609. +static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
  12610. + struct wake_q_head *wake_sleeper_q,
  12611. + struct rt_mutex *lock);
  12612. +/*
  12613. + * Slow path to release a rt_mutex spin_lock style
  12614. + */
  12615. +static void noinline __sched rt_spin_lock_slowunlock(struct rt_mutex *lock)
  12616. +{
  12617. + unsigned long flags;
  12618. + WAKE_Q(wake_q);
  12619. + WAKE_Q(wake_sleeper_q);
  12620. +
  12621. + raw_spin_lock_irqsave(&lock->wait_lock, flags);
  12622. +
  12623. + debug_rt_mutex_unlock(lock);
  12624. +
  12625. + if (!rt_mutex_has_waiters(lock)) {
  12626. + lock->owner = NULL;
  12627. + raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  12628. + return;
  12629. + }
  12630. +
  12631. + mark_wakeup_next_waiter(&wake_q, &wake_sleeper_q, lock);
  12632. +
  12633. + raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  12634. + wake_up_q(&wake_q);
  12635. + wake_up_q_sleeper(&wake_sleeper_q);
  12636. +
  12637. + /* Undo pi boosting.when necessary */
  12638. + rt_mutex_adjust_prio(current);
  12639. +}
  12640. +
  12641. +void __lockfunc rt_spin_lock__no_mg(spinlock_t *lock)
  12642. +{
  12643. + rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock, false);
  12644. + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  12645. +}
  12646. +EXPORT_SYMBOL(rt_spin_lock__no_mg);
  12647. +
  12648. +void __lockfunc rt_spin_lock(spinlock_t *lock)
  12649. +{
  12650. + rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock, true);
  12651. + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  12652. +}
  12653. +EXPORT_SYMBOL(rt_spin_lock);
  12654. +
  12655. +void __lockfunc __rt_spin_lock(struct rt_mutex *lock)
  12656. +{
  12657. + rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock, true);
  12658. +}
  12659. +EXPORT_SYMBOL(__rt_spin_lock);
  12660. +
  12661. +void __lockfunc __rt_spin_lock__no_mg(struct rt_mutex *lock)
  12662. +{
  12663. + rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock, false);
  12664. +}
  12665. +EXPORT_SYMBOL(__rt_spin_lock__no_mg);
  12666. +
  12667. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  12668. +void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass)
  12669. +{
  12670. + spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
  12671. + rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock, true);
  12672. +}
  12673. +EXPORT_SYMBOL(rt_spin_lock_nested);
  12674. +#endif
  12675. +
  12676. +void __lockfunc rt_spin_unlock__no_mg(spinlock_t *lock)
  12677. +{
  12678. + /* NOTE: we always pass in '1' for nested, for simplicity */
  12679. + spin_release(&lock->dep_map, 1, _RET_IP_);
  12680. + rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock);
  12681. +}
  12682. +EXPORT_SYMBOL(rt_spin_unlock__no_mg);
  12683. +
  12684. +void __lockfunc rt_spin_unlock(spinlock_t *lock)
  12685. +{
  12686. + /* NOTE: we always pass in '1' for nested, for simplicity */
  12687. + spin_release(&lock->dep_map, 1, _RET_IP_);
  12688. + rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock);
  12689. + migrate_enable();
  12690. +}
  12691. +EXPORT_SYMBOL(rt_spin_unlock);
  12692. +
  12693. +void __lockfunc __rt_spin_unlock(struct rt_mutex *lock)
  12694. +{
  12695. + rt_spin_lock_fastunlock(lock, rt_spin_lock_slowunlock);
  12696. +}
  12697. +EXPORT_SYMBOL(__rt_spin_unlock);
  12698. +
  12699. +/*
  12700. + * Wait for the lock to get unlocked: instead of polling for an unlock
  12701. + * (like raw spinlocks do), we lock and unlock, to force the kernel to
  12702. + * schedule if there's contention:
  12703. + */
  12704. +void __lockfunc rt_spin_unlock_wait(spinlock_t *lock)
  12705. +{
  12706. + spin_lock(lock);
  12707. + spin_unlock(lock);
  12708. +}
  12709. +EXPORT_SYMBOL(rt_spin_unlock_wait);
  12710. +
  12711. +int __lockfunc rt_spin_trylock__no_mg(spinlock_t *lock)
  12712. +{
  12713. + int ret;
  12714. +
  12715. + ret = rt_mutex_trylock(&lock->lock);
  12716. + if (ret)
  12717. + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  12718. + return ret;
  12719. +}
  12720. +EXPORT_SYMBOL(rt_spin_trylock__no_mg);
  12721. +
  12722. +int __lockfunc rt_spin_trylock(spinlock_t *lock)
  12723. +{
  12724. + int ret;
  12725. +
  12726. + migrate_disable();
  12727. + ret = rt_mutex_trylock(&lock->lock);
  12728. + if (ret)
  12729. + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  12730. + else
  12731. + migrate_enable();
  12732. + return ret;
  12733. +}
  12734. +EXPORT_SYMBOL(rt_spin_trylock);
  12735. +
  12736. +int __lockfunc rt_spin_trylock_bh(spinlock_t *lock)
  12737. +{
  12738. + int ret;
  12739. +
  12740. + local_bh_disable();
  12741. + ret = rt_mutex_trylock(&lock->lock);
  12742. + if (ret) {
  12743. + migrate_disable();
  12744. + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  12745. + } else
  12746. + local_bh_enable();
  12747. + return ret;
  12748. +}
  12749. +EXPORT_SYMBOL(rt_spin_trylock_bh);
  12750. +
  12751. +int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags)
  12752. +{
  12753. + int ret;
  12754. +
  12755. + *flags = 0;
  12756. + ret = rt_mutex_trylock(&lock->lock);
  12757. + if (ret) {
  12758. + migrate_disable();
  12759. + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  12760. + }
  12761. + return ret;
  12762. +}
  12763. +EXPORT_SYMBOL(rt_spin_trylock_irqsave);
  12764. +
  12765. +int atomic_dec_and_spin_lock(atomic_t *atomic, spinlock_t *lock)
  12766. +{
  12767. + /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
  12768. + if (atomic_add_unless(atomic, -1, 1))
  12769. + return 0;
  12770. + rt_spin_lock(lock);
  12771. + if (atomic_dec_and_test(atomic))
  12772. + return 1;
  12773. + rt_spin_unlock(lock);
  12774. + return 0;
  12775. +}
  12776. +EXPORT_SYMBOL(atomic_dec_and_spin_lock);
  12777. +
  12778. + void
  12779. +__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key)
  12780. +{
  12781. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  12782. + /*
  12783. + * Make sure we are not reinitializing a held lock:
  12784. + */
  12785. + debug_check_no_locks_freed((void *)lock, sizeof(*lock));
  12786. + lockdep_init_map(&lock->dep_map, name, key, 0);
  12787. +#endif
  12788. +}
  12789. +EXPORT_SYMBOL(__rt_spin_lock_init);
  12790. +
  12791. +#endif /* PREEMPT_RT_FULL */
  12792. +
  12793. +#ifdef CONFIG_PREEMPT_RT_FULL
  12794. + static inline int __sched
  12795. +__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx)
  12796. +{
  12797. + struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock);
  12798. + struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx);
  12799. +
  12800. + if (!hold_ctx)
  12801. + return 0;
  12802. +
  12803. + if (unlikely(ctx == hold_ctx))
  12804. + return -EALREADY;
  12805. +
  12806. + if (ctx->stamp - hold_ctx->stamp <= LONG_MAX &&
  12807. + (ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) {
  12808. +#ifdef CONFIG_DEBUG_MUTEXES
  12809. + DEBUG_LOCKS_WARN_ON(ctx->contending_lock);
  12810. + ctx->contending_lock = ww;
  12811. +#endif
  12812. + return -EDEADLK;
  12813. + }
  12814. +
  12815. + return 0;
  12816. +}
  12817. +#else
  12818. + static inline int __sched
  12819. +__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx)
  12820. +{
  12821. + BUG();
  12822. + return 0;
  12823. +}
  12824. +
  12825. +#endif
  12826. +
  12827. +static inline int
  12828. +try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
  12829. + struct rt_mutex_waiter *waiter)
  12830. +{
  12831. + return __try_to_take_rt_mutex(lock, task, waiter, STEAL_NORMAL);
  12832. +}
  12833. /*
  12834. * Task blocks on lock.
  12835. @@ -971,6 +1397,23 @@
  12836. return -EDEADLK;
  12837. raw_spin_lock(&task->pi_lock);
  12838. +
  12839. + /*
  12840. + * In the case of futex requeue PI, this will be a proxy
  12841. + * lock. The task will wake unaware that it is enqueueed on
  12842. + * this lock. Avoid blocking on two locks and corrupting
  12843. + * pi_blocked_on via the PI_WAKEUP_INPROGRESS
  12844. + * flag. futex_wait_requeue_pi() sets this when it wakes up
  12845. + * before requeue (due to a signal or timeout). Do not enqueue
  12846. + * the task if PI_WAKEUP_INPROGRESS is set.
  12847. + */
  12848. + if (task != current && task->pi_blocked_on == PI_WAKEUP_INPROGRESS) {
  12849. + raw_spin_unlock(&task->pi_lock);
  12850. + return -EAGAIN;
  12851. + }
  12852. +
  12853. + BUG_ON(rt_mutex_real_waiter(task->pi_blocked_on));
  12854. +
  12855. __rt_mutex_adjust_prio(task);
  12856. waiter->task = task;
  12857. waiter->lock = lock;
  12858. @@ -994,7 +1437,7 @@
  12859. rt_mutex_enqueue_pi(owner, waiter);
  12860. __rt_mutex_adjust_prio(owner);
  12861. - if (owner->pi_blocked_on)
  12862. + if (rt_mutex_real_waiter(owner->pi_blocked_on))
  12863. chain_walk = 1;
  12864. } else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) {
  12865. chain_walk = 1;
  12866. @@ -1036,6 +1479,7 @@
  12867. * Called with lock->wait_lock held and interrupts disabled.
  12868. */
  12869. static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
  12870. + struct wake_q_head *wake_sleeper_q,
  12871. struct rt_mutex *lock)
  12872. {
  12873. struct rt_mutex_waiter *waiter;
  12874. @@ -1064,7 +1508,10 @@
  12875. raw_spin_unlock(&current->pi_lock);
  12876. - wake_q_add(wake_q, waiter->task);
  12877. + if (waiter->savestate)
  12878. + wake_q_add(wake_sleeper_q, waiter->task);
  12879. + else
  12880. + wake_q_add(wake_q, waiter->task);
  12881. }
  12882. /*
  12883. @@ -1078,7 +1525,7 @@
  12884. {
  12885. bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
  12886. struct task_struct *owner = rt_mutex_owner(lock);
  12887. - struct rt_mutex *next_lock;
  12888. + struct rt_mutex *next_lock = NULL;
  12889. raw_spin_lock(&current->pi_lock);
  12890. rt_mutex_dequeue(lock, waiter);
  12891. @@ -1102,7 +1549,8 @@
  12892. __rt_mutex_adjust_prio(owner);
  12893. /* Store the lock on which owner is blocked or NULL */
  12894. - next_lock = task_blocked_on_lock(owner);
  12895. + if (rt_mutex_real_waiter(owner->pi_blocked_on))
  12896. + next_lock = task_blocked_on_lock(owner);
  12897. raw_spin_unlock(&owner->pi_lock);
  12898. @@ -1138,21 +1586,30 @@
  12899. raw_spin_lock_irqsave(&task->pi_lock, flags);
  12900. waiter = task->pi_blocked_on;
  12901. - if (!waiter || (waiter->prio == task->prio &&
  12902. + if (!rt_mutex_real_waiter(waiter) || (waiter->prio == task->prio &&
  12903. !dl_prio(task->prio))) {
  12904. raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  12905. return;
  12906. }
  12907. next_lock = waiter->lock;
  12908. - raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  12909. /* gets dropped in rt_mutex_adjust_prio_chain()! */
  12910. get_task_struct(task);
  12911. + raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  12912. rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL,
  12913. next_lock, NULL, task);
  12914. }
  12915. +void rt_mutex_init_waiter(struct rt_mutex_waiter *waiter, bool savestate)
  12916. +{
  12917. + debug_rt_mutex_init_waiter(waiter);
  12918. + RB_CLEAR_NODE(&waiter->pi_tree_entry);
  12919. + RB_CLEAR_NODE(&waiter->tree_entry);
  12920. + waiter->task = NULL;
  12921. + waiter->savestate = savestate;
  12922. +}
  12923. +
  12924. /**
  12925. * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
  12926. * @lock: the rt_mutex to take
  12927. @@ -1166,7 +1623,8 @@
  12928. static int __sched
  12929. __rt_mutex_slowlock(struct rt_mutex *lock, int state,
  12930. struct hrtimer_sleeper *timeout,
  12931. - struct rt_mutex_waiter *waiter)
  12932. + struct rt_mutex_waiter *waiter,
  12933. + struct ww_acquire_ctx *ww_ctx)
  12934. {
  12935. int ret = 0;
  12936. @@ -1175,16 +1633,17 @@
  12937. if (try_to_take_rt_mutex(lock, current, waiter))
  12938. break;
  12939. - /*
  12940. - * TASK_INTERRUPTIBLE checks for signals and
  12941. - * timeout. Ignored otherwise.
  12942. - */
  12943. - if (unlikely(state == TASK_INTERRUPTIBLE)) {
  12944. - /* Signal pending? */
  12945. - if (signal_pending(current))
  12946. - ret = -EINTR;
  12947. - if (timeout && !timeout->task)
  12948. - ret = -ETIMEDOUT;
  12949. + if (timeout && !timeout->task) {
  12950. + ret = -ETIMEDOUT;
  12951. + break;
  12952. + }
  12953. + if (signal_pending_state(state, current)) {
  12954. + ret = -EINTR;
  12955. + break;
  12956. + }
  12957. +
  12958. + if (ww_ctx && ww_ctx->acquired > 0) {
  12959. + ret = __mutex_lock_check_stamp(lock, ww_ctx);
  12960. if (ret)
  12961. break;
  12962. }
  12963. @@ -1223,35 +1682,94 @@
  12964. }
  12965. }
  12966. -/*
  12967. - * Slow path lock function:
  12968. - */
  12969. -static int __sched
  12970. -rt_mutex_slowlock(struct rt_mutex *lock, int state,
  12971. - struct hrtimer_sleeper *timeout,
  12972. - enum rtmutex_chainwalk chwalk)
  12973. +static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww,
  12974. + struct ww_acquire_ctx *ww_ctx)
  12975. {
  12976. - struct rt_mutex_waiter waiter;
  12977. - unsigned long flags;
  12978. - int ret = 0;
  12979. +#ifdef CONFIG_DEBUG_MUTEXES
  12980. + /*
  12981. + * If this WARN_ON triggers, you used ww_mutex_lock to acquire,
  12982. + * but released with a normal mutex_unlock in this call.
  12983. + *
  12984. + * This should never happen, always use ww_mutex_unlock.
  12985. + */
  12986. + DEBUG_LOCKS_WARN_ON(ww->ctx);
  12987. - debug_rt_mutex_init_waiter(&waiter);
  12988. - RB_CLEAR_NODE(&waiter.pi_tree_entry);
  12989. - RB_CLEAR_NODE(&waiter.tree_entry);
  12990. + /*
  12991. + * Not quite done after calling ww_acquire_done() ?
  12992. + */
  12993. + DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);
  12994. +
  12995. + if (ww_ctx->contending_lock) {
  12996. + /*
  12997. + * After -EDEADLK you tried to
  12998. + * acquire a different ww_mutex? Bad!
  12999. + */
  13000. + DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);
  13001. +
  13002. + /*
  13003. + * You called ww_mutex_lock after receiving -EDEADLK,
  13004. + * but 'forgot' to unlock everything else first?
  13005. + */
  13006. + DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
  13007. + ww_ctx->contending_lock = NULL;
  13008. + }
  13009. /*
  13010. - * Technically we could use raw_spin_[un]lock_irq() here, but this can
  13011. - * be called in early boot if the cmpxchg() fast path is disabled
  13012. - * (debug, no architecture support). In this case we will acquire the
  13013. - * rtmutex with lock->wait_lock held. But we cannot unconditionally
  13014. - * enable interrupts in that early boot case. So we need to use the
  13015. - * irqsave/restore variants.
  13016. + * Naughty, using a different class will lead to undefined behavior!
  13017. */
  13018. - raw_spin_lock_irqsave(&lock->wait_lock, flags);
  13019. + DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
  13020. +#endif
  13021. + ww_ctx->acquired++;
  13022. +}
  13023. +
  13024. +#ifdef CONFIG_PREEMPT_RT_FULL
  13025. +static void ww_mutex_account_lock(struct rt_mutex *lock,
  13026. + struct ww_acquire_ctx *ww_ctx)
  13027. +{
  13028. + struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock);
  13029. + struct rt_mutex_waiter *waiter, *n;
  13030. +
  13031. + /*
  13032. + * This branch gets optimized out for the common case,
  13033. + * and is only important for ww_mutex_lock.
  13034. + */
  13035. + ww_mutex_lock_acquired(ww, ww_ctx);
  13036. + ww->ctx = ww_ctx;
  13037. +
  13038. + /*
  13039. + * Give any possible sleeping processes the chance to wake up,
  13040. + * so they can recheck if they have to back off.
  13041. + */
  13042. + rbtree_postorder_for_each_entry_safe(waiter, n, &lock->waiters,
  13043. + tree_entry) {
  13044. + /* XXX debug rt mutex waiter wakeup */
  13045. +
  13046. + BUG_ON(waiter->lock != lock);
  13047. + rt_mutex_wake_waiter(waiter);
  13048. + }
  13049. +}
  13050. +
  13051. +#else
  13052. +
  13053. +static void ww_mutex_account_lock(struct rt_mutex *lock,
  13054. + struct ww_acquire_ctx *ww_ctx)
  13055. +{
  13056. + BUG();
  13057. +}
  13058. +#endif
  13059. +
  13060. +int __sched rt_mutex_slowlock_locked(struct rt_mutex *lock, int state,
  13061. + struct hrtimer_sleeper *timeout,
  13062. + enum rtmutex_chainwalk chwalk,
  13063. + struct ww_acquire_ctx *ww_ctx,
  13064. + struct rt_mutex_waiter *waiter)
  13065. +{
  13066. + int ret;
  13067. /* Try to acquire the lock again: */
  13068. if (try_to_take_rt_mutex(lock, current, NULL)) {
  13069. - raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  13070. + if (ww_ctx)
  13071. + ww_mutex_account_lock(lock, ww_ctx);
  13072. return 0;
  13073. }
  13074. @@ -1261,17 +1779,27 @@
  13075. if (unlikely(timeout))
  13076. hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
  13077. - ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk);
  13078. + ret = task_blocks_on_rt_mutex(lock, waiter, current, chwalk);
  13079. - if (likely(!ret))
  13080. + if (likely(!ret)) {
  13081. /* sleep on the mutex */
  13082. - ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
  13083. + ret = __rt_mutex_slowlock(lock, state, timeout, waiter,
  13084. + ww_ctx);
  13085. + } else if (ww_ctx) {
  13086. + /* ww_mutex received EDEADLK, let it become EALREADY */
  13087. + ret = __mutex_lock_check_stamp(lock, ww_ctx);
  13088. + BUG_ON(!ret);
  13089. + }
  13090. if (unlikely(ret)) {
  13091. __set_current_state(TASK_RUNNING);
  13092. if (rt_mutex_has_waiters(lock))
  13093. - remove_waiter(lock, &waiter);
  13094. - rt_mutex_handle_deadlock(ret, chwalk, &waiter);
  13095. + remove_waiter(lock, waiter);
  13096. + /* ww_mutex want to report EDEADLK/EALREADY, let them */
  13097. + if (!ww_ctx)
  13098. + rt_mutex_handle_deadlock(ret, chwalk, waiter);
  13099. + } else if (ww_ctx) {
  13100. + ww_mutex_account_lock(lock, ww_ctx);
  13101. }
  13102. /*
  13103. @@ -1279,6 +1807,36 @@
  13104. * unconditionally. We might have to fix that up.
  13105. */
  13106. fixup_rt_mutex_waiters(lock);
  13107. + return ret;
  13108. +}
  13109. +
  13110. +/*
  13111. + * Slow path lock function:
  13112. + */
  13113. +static int __sched
  13114. +rt_mutex_slowlock(struct rt_mutex *lock, int state,
  13115. + struct hrtimer_sleeper *timeout,
  13116. + enum rtmutex_chainwalk chwalk,
  13117. + struct ww_acquire_ctx *ww_ctx)
  13118. +{
  13119. + struct rt_mutex_waiter waiter;
  13120. + unsigned long flags;
  13121. + int ret = 0;
  13122. +
  13123. + rt_mutex_init_waiter(&waiter, false);
  13124. +
  13125. + /*
  13126. + * Technically we could use raw_spin_[un]lock_irq() here, but this can
  13127. + * be called in early boot if the cmpxchg() fast path is disabled
  13128. + * (debug, no architecture support). In this case we will acquire the
  13129. + * rtmutex with lock->wait_lock held. But we cannot unconditionally
  13130. + * enable interrupts in that early boot case. So we need to use the
  13131. + * irqsave/restore variants.
  13132. + */
  13133. + raw_spin_lock_irqsave(&lock->wait_lock, flags);
  13134. +
  13135. + ret = rt_mutex_slowlock_locked(lock, state, timeout, chwalk, ww_ctx,
  13136. + &waiter);
  13137. raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  13138. @@ -1331,7 +1889,8 @@
  13139. * Return whether the current task needs to undo a potential priority boosting.
  13140. */
  13141. static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock,
  13142. - struct wake_q_head *wake_q)
  13143. + struct wake_q_head *wake_q,
  13144. + struct wake_q_head *wake_sleeper_q)
  13145. {
  13146. unsigned long flags;
  13147. @@ -1340,8 +1899,6 @@
  13148. debug_rt_mutex_unlock(lock);
  13149. - rt_mutex_deadlock_account_unlock(current);
  13150. -
  13151. /*
  13152. * We must be careful here if the fast path is enabled. If we
  13153. * have no waiters queued we cannot set owner to NULL here
  13154. @@ -1387,7 +1944,7 @@
  13155. *
  13156. * Queue the next waiter for wakeup once we release the wait_lock.
  13157. */
  13158. - mark_wakeup_next_waiter(wake_q, lock);
  13159. + mark_wakeup_next_waiter(wake_q, wake_sleeper_q, lock);
  13160. raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  13161. @@ -1403,63 +1960,79 @@
  13162. */
  13163. static inline int
  13164. rt_mutex_fastlock(struct rt_mutex *lock, int state,
  13165. + struct ww_acquire_ctx *ww_ctx,
  13166. int (*slowfn)(struct rt_mutex *lock, int state,
  13167. struct hrtimer_sleeper *timeout,
  13168. - enum rtmutex_chainwalk chwalk))
  13169. + enum rtmutex_chainwalk chwalk,
  13170. + struct ww_acquire_ctx *ww_ctx))
  13171. {
  13172. - if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) {
  13173. - rt_mutex_deadlock_account_lock(lock, current);
  13174. + if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
  13175. return 0;
  13176. - } else
  13177. - return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);
  13178. +
  13179. + return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK, ww_ctx);
  13180. }
  13181. static inline int
  13182. rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
  13183. struct hrtimer_sleeper *timeout,
  13184. enum rtmutex_chainwalk chwalk,
  13185. + struct ww_acquire_ctx *ww_ctx,
  13186. int (*slowfn)(struct rt_mutex *lock, int state,
  13187. struct hrtimer_sleeper *timeout,
  13188. - enum rtmutex_chainwalk chwalk))
  13189. + enum rtmutex_chainwalk chwalk,
  13190. + struct ww_acquire_ctx *ww_ctx))
  13191. {
  13192. if (chwalk == RT_MUTEX_MIN_CHAINWALK &&
  13193. - likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) {
  13194. - rt_mutex_deadlock_account_lock(lock, current);
  13195. + likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
  13196. return 0;
  13197. - } else
  13198. - return slowfn(lock, state, timeout, chwalk);
  13199. +
  13200. + return slowfn(lock, state, timeout, chwalk, ww_ctx);
  13201. }
  13202. static inline int
  13203. rt_mutex_fasttrylock(struct rt_mutex *lock,
  13204. int (*slowfn)(struct rt_mutex *lock))
  13205. {
  13206. - if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) {
  13207. - rt_mutex_deadlock_account_lock(lock, current);
  13208. + if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
  13209. return 1;
  13210. - }
  13211. +
  13212. return slowfn(lock);
  13213. }
  13214. static inline void
  13215. rt_mutex_fastunlock(struct rt_mutex *lock,
  13216. bool (*slowfn)(struct rt_mutex *lock,
  13217. - struct wake_q_head *wqh))
  13218. + struct wake_q_head *wqh,
  13219. + struct wake_q_head *wq_sleeper))
  13220. {
  13221. WAKE_Q(wake_q);
  13222. + WAKE_Q(wake_sleeper_q);
  13223. + bool deboost;
  13224. - if (likely(rt_mutex_cmpxchg_release(lock, current, NULL))) {
  13225. - rt_mutex_deadlock_account_unlock(current);
  13226. + if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))
  13227. + return;
  13228. - } else {
  13229. - bool deboost = slowfn(lock, &wake_q);
  13230. + deboost = slowfn(lock, &wake_q, &wake_sleeper_q);
  13231. - wake_up_q(&wake_q);
  13232. + wake_up_q(&wake_q);
  13233. + wake_up_q_sleeper(&wake_sleeper_q);
  13234. - /* Undo pi boosting if necessary: */
  13235. - if (deboost)
  13236. - rt_mutex_adjust_prio(current);
  13237. - }
  13238. + /* Undo pi boosting if necessary: */
  13239. + if (deboost)
  13240. + rt_mutex_adjust_prio(current);
  13241. +}
  13242. +
  13243. +/**
  13244. + * rt_mutex_lock_state - lock a rt_mutex with a given state
  13245. + *
  13246. + * @lock: The rt_mutex to be locked
  13247. + * @state: The state to set when blocking on the rt_mutex
  13248. + */
  13249. +int __sched rt_mutex_lock_state(struct rt_mutex *lock, int state)
  13250. +{
  13251. + might_sleep();
  13252. +
  13253. + return rt_mutex_fastlock(lock, state, NULL, rt_mutex_slowlock);
  13254. }
  13255. /**
  13256. @@ -1469,15 +2042,13 @@
  13257. */
  13258. void __sched rt_mutex_lock(struct rt_mutex *lock)
  13259. {
  13260. - might_sleep();
  13261. -
  13262. - rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock);
  13263. + rt_mutex_lock_state(lock, TASK_UNINTERRUPTIBLE);
  13264. }
  13265. EXPORT_SYMBOL_GPL(rt_mutex_lock);
  13266. /**
  13267. * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
  13268. - *
  13269. + **
  13270. * @lock: the rt_mutex to be locked
  13271. *
  13272. * Returns:
  13273. @@ -1486,23 +2057,32 @@
  13274. */
  13275. int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock)
  13276. {
  13277. - might_sleep();
  13278. -
  13279. - return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock);
  13280. + return rt_mutex_lock_state(lock, TASK_INTERRUPTIBLE);
  13281. }
  13282. EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
  13283. -/*
  13284. - * Futex variant with full deadlock detection.
  13285. +/**
  13286. + * rt_mutex_lock_killable - lock a rt_mutex killable
  13287. + *
  13288. + * @lock: the rt_mutex to be locked
  13289. + * @detect_deadlock: deadlock detection on/off
  13290. + *
  13291. + * Returns:
  13292. + * 0 on success
  13293. + * -EINTR when interrupted by a signal
  13294. */
  13295. -int rt_mutex_timed_futex_lock(struct rt_mutex *lock,
  13296. - struct hrtimer_sleeper *timeout)
  13297. +int __sched rt_mutex_lock_killable(struct rt_mutex *lock)
  13298. {
  13299. - might_sleep();
  13300. + return rt_mutex_lock_state(lock, TASK_KILLABLE);
  13301. +}
  13302. +EXPORT_SYMBOL_GPL(rt_mutex_lock_killable);
  13303. - return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
  13304. - RT_MUTEX_FULL_CHAINWALK,
  13305. - rt_mutex_slowlock);
  13306. +/*
  13307. + * Futex variant, must not use fastpath.
  13308. + */
  13309. +int __sched rt_mutex_futex_trylock(struct rt_mutex *lock)
  13310. +{
  13311. + return rt_mutex_slowtrylock(lock);
  13312. }
  13313. /**
  13314. @@ -1525,6 +2105,7 @@
  13315. return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
  13316. RT_MUTEX_MIN_CHAINWALK,
  13317. + NULL,
  13318. rt_mutex_slowlock);
  13319. }
  13320. EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
  13321. @@ -1542,7 +2123,11 @@
  13322. */
  13323. int __sched rt_mutex_trylock(struct rt_mutex *lock)
  13324. {
  13325. +#ifdef CONFIG_PREEMPT_RT_FULL
  13326. + if (WARN_ON_ONCE(in_irq() || in_nmi()))
  13327. +#else
  13328. if (WARN_ON_ONCE(in_irq() || in_nmi() || in_serving_softirq()))
  13329. +#endif
  13330. return 0;
  13331. return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
  13332. @@ -1561,20 +2146,41 @@
  13333. EXPORT_SYMBOL_GPL(rt_mutex_unlock);
  13334. /**
  13335. - * rt_mutex_futex_unlock - Futex variant of rt_mutex_unlock
  13336. - * @lock: the rt_mutex to be unlocked
  13337. - *
  13338. - * Returns: true/false indicating whether priority adjustment is
  13339. - * required or not.
  13340. + * Futex variant, that since futex variants do not use the fast-path, can be
  13341. + * simple and will not need to retry.
  13342. */
  13343. -bool __sched rt_mutex_futex_unlock(struct rt_mutex *lock,
  13344. - struct wake_q_head *wqh)
  13345. +bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock,
  13346. + struct wake_q_head *wake_q,
  13347. + struct wake_q_head *wq_sleeper)
  13348. +{
  13349. + lockdep_assert_held(&lock->wait_lock);
  13350. +
  13351. + debug_rt_mutex_unlock(lock);
  13352. +
  13353. + if (!rt_mutex_has_waiters(lock)) {
  13354. + lock->owner = NULL;
  13355. + return false; /* done */
  13356. + }
  13357. +
  13358. + mark_wakeup_next_waiter(wake_q, wq_sleeper, lock);
  13359. + return true; /* deboost and wakeups */
  13360. +}
  13361. +
  13362. +void __sched rt_mutex_futex_unlock(struct rt_mutex *lock)
  13363. {
  13364. - if (likely(rt_mutex_cmpxchg_release(lock, current, NULL))) {
  13365. - rt_mutex_deadlock_account_unlock(current);
  13366. - return false;
  13367. + WAKE_Q(wake_q);
  13368. + WAKE_Q(wake_sleeper_q);
  13369. + bool deboost;
  13370. +
  13371. + raw_spin_lock_irq(&lock->wait_lock);
  13372. + deboost = __rt_mutex_futex_unlock(lock, &wake_q, &wake_sleeper_q);
  13373. + raw_spin_unlock_irq(&lock->wait_lock);
  13374. +
  13375. + if (deboost) {
  13376. + wake_up_q(&wake_q);
  13377. + wake_up_q_sleeper(&wake_sleeper_q);
  13378. + rt_mutex_adjust_prio(current);
  13379. }
  13380. - return rt_mutex_slowunlock(lock, wqh);
  13381. }
  13382. /**
  13383. @@ -1607,13 +2213,12 @@
  13384. void __rt_mutex_init(struct rt_mutex *lock, const char *name)
  13385. {
  13386. lock->owner = NULL;
  13387. - raw_spin_lock_init(&lock->wait_lock);
  13388. lock->waiters = RB_ROOT;
  13389. lock->waiters_leftmost = NULL;
  13390. debug_rt_mutex_init(lock, name);
  13391. }
  13392. -EXPORT_SYMBOL_GPL(__rt_mutex_init);
  13393. +EXPORT_SYMBOL(__rt_mutex_init);
  13394. /**
  13395. * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
  13396. @@ -1628,10 +2233,9 @@
  13397. void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
  13398. struct task_struct *proxy_owner)
  13399. {
  13400. - __rt_mutex_init(lock, NULL);
  13401. + rt_mutex_init(lock);
  13402. debug_rt_mutex_proxy_lock(lock, proxy_owner);
  13403. rt_mutex_set_owner(lock, proxy_owner);
  13404. - rt_mutex_deadlock_account_lock(lock, proxy_owner);
  13405. }
  13406. /**
  13407. @@ -1647,34 +2251,45 @@
  13408. {
  13409. debug_rt_mutex_proxy_unlock(lock);
  13410. rt_mutex_set_owner(lock, NULL);
  13411. - rt_mutex_deadlock_account_unlock(proxy_owner);
  13412. }
  13413. -/**
  13414. - * rt_mutex_start_proxy_lock() - Start lock acquisition for another task
  13415. - * @lock: the rt_mutex to take
  13416. - * @waiter: the pre-initialized rt_mutex_waiter
  13417. - * @task: the task to prepare
  13418. - *
  13419. - * Returns:
  13420. - * 0 - task blocked on lock
  13421. - * 1 - acquired the lock for task, caller should wake it up
  13422. - * <0 - error
  13423. - *
  13424. - * Special API call for FUTEX_REQUEUE_PI support.
  13425. - */
  13426. -int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
  13427. +int __rt_mutex_start_proxy_lock(struct rt_mutex *lock,
  13428. struct rt_mutex_waiter *waiter,
  13429. struct task_struct *task)
  13430. {
  13431. int ret;
  13432. - raw_spin_lock_irq(&lock->wait_lock);
  13433. + if (try_to_take_rt_mutex(lock, task, NULL))
  13434. + return 1;
  13435. - if (try_to_take_rt_mutex(lock, task, NULL)) {
  13436. +#ifdef CONFIG_PREEMPT_RT_FULL
  13437. + /*
  13438. + * In PREEMPT_RT there's an added race.
  13439. + * If the task, that we are about to requeue, times out,
  13440. + * it can set the PI_WAKEUP_INPROGRESS. This tells the requeue
  13441. + * to skip this task. But right after the task sets
  13442. + * its pi_blocked_on to PI_WAKEUP_INPROGRESS it can then
  13443. + * block on the spin_lock(&hb->lock), which in RT is an rtmutex.
  13444. + * This will replace the PI_WAKEUP_INPROGRESS with the actual
  13445. + * lock that it blocks on. We *must not* place this task
  13446. + * on this proxy lock in that case.
  13447. + *
  13448. + * To prevent this race, we first take the task's pi_lock
  13449. + * and check if it has updated its pi_blocked_on. If it has,
  13450. + * we assume that it woke up and we return -EAGAIN.
  13451. + * Otherwise, we set the task's pi_blocked_on to
  13452. + * PI_REQUEUE_INPROGRESS, so that if the task is waking up
  13453. + * it will know that we are in the process of requeuing it.
  13454. + */
  13455. + raw_spin_lock(&task->pi_lock);
  13456. + if (task->pi_blocked_on) {
  13457. + raw_spin_unlock(&task->pi_lock);
  13458. raw_spin_unlock_irq(&lock->wait_lock);
  13459. - return 1;
  13460. + return -EAGAIN;
  13461. }
  13462. + task->pi_blocked_on = PI_REQUEUE_INPROGRESS;
  13463. + raw_spin_unlock(&task->pi_lock);
  13464. +#endif
  13465. /* We enforce deadlock detection for futexes */
  13466. ret = task_blocks_on_rt_mutex(lock, waiter, task,
  13467. @@ -1690,17 +2305,41 @@
  13468. ret = 0;
  13469. }
  13470. - if (unlikely(ret))
  13471. + if (ret && rt_mutex_has_waiters(lock))
  13472. remove_waiter(lock, waiter);
  13473. - raw_spin_unlock_irq(&lock->wait_lock);
  13474. -
  13475. debug_rt_mutex_print_deadlock(waiter);
  13476. return ret;
  13477. }
  13478. /**
  13479. + * rt_mutex_start_proxy_lock() - Start lock acquisition for another task
  13480. + * @lock: the rt_mutex to take
  13481. + * @waiter: the pre-initialized rt_mutex_waiter
  13482. + * @task: the task to prepare
  13483. + *
  13484. + * Returns:
  13485. + * 0 - task blocked on lock
  13486. + * 1 - acquired the lock for task, caller should wake it up
  13487. + * <0 - error
  13488. + *
  13489. + * Special API call for FUTEX_REQUEUE_PI support.
  13490. + */
  13491. +int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
  13492. + struct rt_mutex_waiter *waiter,
  13493. + struct task_struct *task)
  13494. +{
  13495. + int ret;
  13496. +
  13497. + raw_spin_lock_irq(&lock->wait_lock);
  13498. + ret = __rt_mutex_start_proxy_lock(lock, waiter, task);
  13499. + raw_spin_unlock_irq(&lock->wait_lock);
  13500. +
  13501. + return ret;
  13502. +}
  13503. +
  13504. +/**
  13505. * rt_mutex_next_owner - return the next owner of the lock
  13506. *
  13507. * @lock: the rt lock query
  13508. @@ -1721,21 +2360,23 @@
  13509. }
  13510. /**
  13511. - * rt_mutex_finish_proxy_lock() - Complete lock acquisition
  13512. + * rt_mutex_wait_proxy_lock() - Wait for lock acquisition
  13513. * @lock: the rt_mutex we were woken on
  13514. * @to: the timeout, null if none. hrtimer should already have
  13515. * been started.
  13516. * @waiter: the pre-initialized rt_mutex_waiter
  13517. *
  13518. - * Complete the lock acquisition started our behalf by another thread.
  13519. + * Wait for the the lock acquisition started on our behalf by
  13520. + * rt_mutex_start_proxy_lock(). Upon failure, the caller must call
  13521. + * rt_mutex_cleanup_proxy_lock().
  13522. *
  13523. * Returns:
  13524. * 0 - success
  13525. * <0 - error, one of -EINTR, -ETIMEDOUT
  13526. *
  13527. - * Special API call for PI-futex requeue support
  13528. + * Special API call for PI-futex support
  13529. */
  13530. -int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
  13531. +int rt_mutex_wait_proxy_lock(struct rt_mutex *lock,
  13532. struct hrtimer_sleeper *to,
  13533. struct rt_mutex_waiter *waiter)
  13534. {
  13535. @@ -1746,10 +2387,47 @@
  13536. set_current_state(TASK_INTERRUPTIBLE);
  13537. /* sleep on the mutex */
  13538. - ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
  13539. + ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter, NULL);
  13540. - if (unlikely(ret))
  13541. + raw_spin_unlock_irq(&lock->wait_lock);
  13542. +
  13543. + return ret;
  13544. +}
  13545. +
  13546. +/**
  13547. + * rt_mutex_cleanup_proxy_lock() - Cleanup failed lock acquisition
  13548. + * @lock: the rt_mutex we were woken on
  13549. + * @waiter: the pre-initialized rt_mutex_waiter
  13550. + *
  13551. + * Attempt to clean up after a failed rt_mutex_wait_proxy_lock().
  13552. + *
  13553. + * Unless we acquired the lock; we're still enqueued on the wait-list and can
  13554. + * in fact still be granted ownership until we're removed. Therefore we can
  13555. + * find we are in fact the owner and must disregard the
  13556. + * rt_mutex_wait_proxy_lock() failure.
  13557. + *
  13558. + * Returns:
  13559. + * true - did the cleanup, we done.
  13560. + * false - we acquired the lock after rt_mutex_wait_proxy_lock() returned,
  13561. + * caller should disregards its return value.
  13562. + *
  13563. + * Special API call for PI-futex support
  13564. + */
  13565. +bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock,
  13566. + struct rt_mutex_waiter *waiter)
  13567. +{
  13568. + bool cleanup = false;
  13569. +
  13570. + raw_spin_lock_irq(&lock->wait_lock);
  13571. + /*
  13572. + * Unless we're the owner; we're still enqueued on the wait_list.
  13573. + * So check if we became owner, if not, take us off the wait_list.
  13574. + */
  13575. + if (rt_mutex_owner(lock) != current) {
  13576. remove_waiter(lock, waiter);
  13577. + fixup_rt_mutex_waiters(lock);
  13578. + cleanup = true;
  13579. + }
  13580. /*
  13581. * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might
  13582. @@ -1759,5 +2437,91 @@
  13583. raw_spin_unlock_irq(&lock->wait_lock);
  13584. + return cleanup;
  13585. +}
  13586. +
  13587. +static inline int
  13588. +ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  13589. +{
  13590. +#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
  13591. + unsigned tmp;
  13592. +
  13593. + if (ctx->deadlock_inject_countdown-- == 0) {
  13594. + tmp = ctx->deadlock_inject_interval;
  13595. + if (tmp > UINT_MAX/4)
  13596. + tmp = UINT_MAX;
  13597. + else
  13598. + tmp = tmp*2 + tmp + tmp/2;
  13599. +
  13600. + ctx->deadlock_inject_interval = tmp;
  13601. + ctx->deadlock_inject_countdown = tmp;
  13602. + ctx->contending_lock = lock;
  13603. +
  13604. + ww_mutex_unlock(lock);
  13605. +
  13606. + return -EDEADLK;
  13607. + }
  13608. +#endif
  13609. +
  13610. + return 0;
  13611. +}
  13612. +
  13613. +#ifdef CONFIG_PREEMPT_RT_FULL
  13614. +int __sched
  13615. +__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx)
  13616. +{
  13617. + int ret;
  13618. +
  13619. + might_sleep();
  13620. +
  13621. + mutex_acquire_nest(&lock->base.dep_map, 0, 0, &ww_ctx->dep_map, _RET_IP_);
  13622. + ret = rt_mutex_slowlock(&lock->base.lock, TASK_INTERRUPTIBLE, NULL, 0, ww_ctx);
  13623. + if (ret)
  13624. + mutex_release(&lock->base.dep_map, 1, _RET_IP_);
  13625. + else if (!ret && ww_ctx->acquired > 1)
  13626. + return ww_mutex_deadlock_injection(lock, ww_ctx);
  13627. +
  13628. return ret;
  13629. }
  13630. +EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible);
  13631. +
  13632. +int __sched
  13633. +__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx)
  13634. +{
  13635. + int ret;
  13636. +
  13637. + might_sleep();
  13638. +
  13639. + mutex_acquire_nest(&lock->base.dep_map, 0, 0, &ww_ctx->dep_map, _RET_IP_);
  13640. + ret = rt_mutex_slowlock(&lock->base.lock, TASK_UNINTERRUPTIBLE, NULL, 0, ww_ctx);
  13641. + if (ret)
  13642. + mutex_release(&lock->base.dep_map, 1, _RET_IP_);
  13643. + else if (!ret && ww_ctx->acquired > 1)
  13644. + return ww_mutex_deadlock_injection(lock, ww_ctx);
  13645. +
  13646. + return ret;
  13647. +}
  13648. +EXPORT_SYMBOL_GPL(__ww_mutex_lock);
  13649. +
  13650. +void __sched ww_mutex_unlock(struct ww_mutex *lock)
  13651. +{
  13652. + int nest = !!lock->ctx;
  13653. +
  13654. + /*
  13655. + * The unlocking fastpath is the 0->1 transition from 'locked'
  13656. + * into 'unlocked' state:
  13657. + */
  13658. + if (nest) {
  13659. +#ifdef CONFIG_DEBUG_MUTEXES
  13660. + DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired);
  13661. +#endif
  13662. + if (lock->ctx->acquired > 0)
  13663. + lock->ctx->acquired--;
  13664. + lock->ctx = NULL;
  13665. + }
  13666. +
  13667. + mutex_release(&lock->base.dep_map, nest, _RET_IP_);
  13668. + rt_mutex_unlock(&lock->base.lock);
  13669. +}
  13670. +EXPORT_SYMBOL(ww_mutex_unlock);
  13671. +#endif
  13672. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rtmutex.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rtmutex.h
  13673. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rtmutex.h 2017-04-16 10:38:29.000000000 +0200
  13674. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rtmutex.h 2017-04-18 17:54:26.000000000 +0200
  13675. @@ -11,8 +11,6 @@
  13676. */
  13677. #define rt_mutex_deadlock_check(l) (0)
  13678. -#define rt_mutex_deadlock_account_lock(m, t) do { } while (0)
  13679. -#define rt_mutex_deadlock_account_unlock(l) do { } while (0)
  13680. #define debug_rt_mutex_init_waiter(w) do { } while (0)
  13681. #define debug_rt_mutex_free_waiter(w) do { } while (0)
  13682. #define debug_rt_mutex_lock(l) do { } while (0)
  13683. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rtmutex_common.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rtmutex_common.h
  13684. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rtmutex_common.h 2017-04-16 10:38:29.000000000 +0200
  13685. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rtmutex_common.h 2017-04-18 17:54:26.000000000 +0200
  13686. @@ -27,6 +27,7 @@
  13687. struct rb_node pi_tree_entry;
  13688. struct task_struct *task;
  13689. struct rt_mutex *lock;
  13690. + bool savestate;
  13691. #ifdef CONFIG_DEBUG_RT_MUTEXES
  13692. unsigned long ip;
  13693. struct pid *deadlock_task_pid;
  13694. @@ -98,22 +99,45 @@
  13695. /*
  13696. * PI-futex support (proxy locking functions, etc.):
  13697. */
  13698. +#define PI_WAKEUP_INPROGRESS ((struct rt_mutex_waiter *) 1)
  13699. +#define PI_REQUEUE_INPROGRESS ((struct rt_mutex_waiter *) 2)
  13700. +
  13701. extern struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock);
  13702. extern void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
  13703. struct task_struct *proxy_owner);
  13704. extern void rt_mutex_proxy_unlock(struct rt_mutex *lock,
  13705. struct task_struct *proxy_owner);
  13706. +extern void rt_mutex_init_waiter(struct rt_mutex_waiter *waiter, bool savetate);
  13707. +extern int __rt_mutex_start_proxy_lock(struct rt_mutex *lock,
  13708. + struct rt_mutex_waiter *waiter,
  13709. + struct task_struct *task);
  13710. extern int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
  13711. struct rt_mutex_waiter *waiter,
  13712. struct task_struct *task);
  13713. -extern int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
  13714. - struct hrtimer_sleeper *to,
  13715. - struct rt_mutex_waiter *waiter);
  13716. -extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to);
  13717. -extern bool rt_mutex_futex_unlock(struct rt_mutex *lock,
  13718. - struct wake_q_head *wqh);
  13719. +extern int rt_mutex_wait_proxy_lock(struct rt_mutex *lock,
  13720. + struct hrtimer_sleeper *to,
  13721. + struct rt_mutex_waiter *waiter);
  13722. +extern bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock,
  13723. + struct rt_mutex_waiter *waiter);
  13724. +
  13725. +extern int rt_mutex_futex_trylock(struct rt_mutex *l);
  13726. +
  13727. +extern void rt_mutex_futex_unlock(struct rt_mutex *lock);
  13728. +extern bool __rt_mutex_futex_unlock(struct rt_mutex *lock,
  13729. + struct wake_q_head *wqh,
  13730. + struct wake_q_head *wq_sleeper);
  13731. +
  13732. extern void rt_mutex_adjust_prio(struct task_struct *task);
  13733. +/* RW semaphore special interface */
  13734. +struct ww_acquire_ctx;
  13735. +
  13736. +int __sched rt_mutex_slowlock_locked(struct rt_mutex *lock, int state,
  13737. + struct hrtimer_sleeper *timeout,
  13738. + enum rtmutex_chainwalk chwalk,
  13739. + struct ww_acquire_ctx *ww_ctx,
  13740. + struct rt_mutex_waiter *waiter);
  13741. +
  13742. #ifdef CONFIG_DEBUG_RT_MUTEXES
  13743. # include "rtmutex-debug.h"
  13744. #else
  13745. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rwsem-rt.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rwsem-rt.c
  13746. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/rwsem-rt.c 1970-01-01 01:00:00.000000000 +0100
  13747. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/rwsem-rt.c 2017-04-18 17:54:26.000000000 +0200
  13748. @@ -0,0 +1,268 @@
  13749. +/*
  13750. + */
  13751. +#include <linux/rwsem.h>
  13752. +#include <linux/sched.h>
  13753. +#include <linux/export.h>
  13754. +
  13755. +#include "rtmutex_common.h"
  13756. +
  13757. +/*
  13758. + * RT-specific reader/writer semaphores
  13759. + *
  13760. + * down_write()
  13761. + * 1) Lock sem->rtmutex
  13762. + * 2) Remove the reader BIAS to force readers into the slow path
  13763. + * 3) Wait until all readers have left the critical region
  13764. + * 4) Mark it write locked
  13765. + *
  13766. + * up_write()
  13767. + * 1) Remove the write locked marker
  13768. + * 2) Set the reader BIAS so readers can use the fast path again
  13769. + * 3) Unlock sem->rtmutex to release blocked readers
  13770. + *
  13771. + * down_read()
  13772. + * 1) Try fast path acquisition (reader BIAS is set)
  13773. + * 2) Take sem->rtmutex.wait_lock which protects the writelocked flag
  13774. + * 3) If !writelocked, acquire it for read
  13775. + * 4) If writelocked, block on sem->rtmutex
  13776. + * 5) unlock sem->rtmutex, goto 1)
  13777. + *
  13778. + * up_read()
  13779. + * 1) Try fast path release (reader count != 1)
  13780. + * 2) Wake the writer waiting in down_write()#3
  13781. + *
  13782. + * down_read()#3 has the consequence, that rw semaphores on RT are not writer
  13783. + * fair, but writers, which should be avoided in RT tasks (think mmap_sem),
  13784. + * are subject to the rtmutex priority/DL inheritance mechanism.
  13785. + *
  13786. + * It's possible to make the rw semaphores writer fair by keeping a list of
  13787. + * active readers. A blocked writer would force all newly incoming readers to
  13788. + * block on the rtmutex, but the rtmutex would have to be proxy locked for one
  13789. + * reader after the other. We can't use multi-reader inheritance because there
  13790. + * is no way to support that with SCHED_DEADLINE. Implementing the one by one
  13791. + * reader boosting/handover mechanism is a major surgery for a very dubious
  13792. + * value.
  13793. + *
  13794. + * The risk of writer starvation is there, but the pathological use cases
  13795. + * which trigger it are not necessarily the typical RT workloads.
  13796. + */
  13797. +
  13798. +void __rwsem_init(struct rw_semaphore *sem, const char *name,
  13799. + struct lock_class_key *key)
  13800. +{
  13801. +#ifdef CONFIG_DEBUG_LOCK_ALLOC
  13802. + /*
  13803. + * Make sure we are not reinitializing a held semaphore:
  13804. + */
  13805. + debug_check_no_locks_freed((void *)sem, sizeof(*sem));
  13806. + lockdep_init_map(&sem->dep_map, name, key, 0);
  13807. +#endif
  13808. + atomic_set(&sem->readers, READER_BIAS);
  13809. +}
  13810. +EXPORT_SYMBOL(__rwsem_init);
  13811. +
  13812. +int __down_read_trylock(struct rw_semaphore *sem)
  13813. +{
  13814. + int r, old;
  13815. +
  13816. + /*
  13817. + * Increment reader count, if sem->readers < 0, i.e. READER_BIAS is
  13818. + * set.
  13819. + */
  13820. + for (r = atomic_read(&sem->readers); r < 0;) {
  13821. + old = atomic_cmpxchg(&sem->readers, r, r + 1);
  13822. + if (likely(old == r))
  13823. + return 1;
  13824. + r = old;
  13825. + }
  13826. + return 0;
  13827. +}
  13828. +
  13829. +void __sched __down_read(struct rw_semaphore *sem)
  13830. +{
  13831. + struct rt_mutex *m = &sem->rtmutex;
  13832. + struct rt_mutex_waiter waiter;
  13833. +
  13834. + if (__down_read_trylock(sem))
  13835. + return;
  13836. +
  13837. + might_sleep();
  13838. + raw_spin_lock_irq(&m->wait_lock);
  13839. + /*
  13840. + * Allow readers as long as the writer has not completely
  13841. + * acquired the semaphore for write.
  13842. + */
  13843. + if (atomic_read(&sem->readers) != WRITER_BIAS) {
  13844. + atomic_inc(&sem->readers);
  13845. + raw_spin_unlock_irq(&m->wait_lock);
  13846. + return;
  13847. + }
  13848. +
  13849. + /*
  13850. + * Call into the slow lock path with the rtmutex->wait_lock
  13851. + * held, so this can't result in the following race:
  13852. + *
  13853. + * Reader1 Reader2 Writer
  13854. + * down_read()
  13855. + * down_write()
  13856. + * rtmutex_lock(m)
  13857. + * swait()
  13858. + * down_read()
  13859. + * unlock(m->wait_lock)
  13860. + * up_read()
  13861. + * swake()
  13862. + * lock(m->wait_lock)
  13863. + * sem->writelocked=true
  13864. + * unlock(m->wait_lock)
  13865. + *
  13866. + * up_write()
  13867. + * sem->writelocked=false
  13868. + * rtmutex_unlock(m)
  13869. + * down_read()
  13870. + * down_write()
  13871. + * rtmutex_lock(m)
  13872. + * swait()
  13873. + * rtmutex_lock(m)
  13874. + *
  13875. + * That would put Reader1 behind the writer waiting on
  13876. + * Reader2 to call up_read() which might be unbound.
  13877. + */
  13878. + rt_mutex_init_waiter(&waiter, false);
  13879. + rt_mutex_slowlock_locked(m, TASK_UNINTERRUPTIBLE, NULL,
  13880. + RT_MUTEX_MIN_CHAINWALK, NULL,
  13881. + &waiter);
  13882. + /*
  13883. + * The slowlock() above is guaranteed to return with the rtmutex is
  13884. + * now held, so there can't be a writer active. Increment the reader
  13885. + * count and immediately drop the rtmutex again.
  13886. + */
  13887. + atomic_inc(&sem->readers);
  13888. + raw_spin_unlock_irq(&m->wait_lock);
  13889. + rt_mutex_unlock(m);
  13890. +
  13891. + debug_rt_mutex_free_waiter(&waiter);
  13892. +}
  13893. +
  13894. +void __up_read(struct rw_semaphore *sem)
  13895. +{
  13896. + struct rt_mutex *m = &sem->rtmutex;
  13897. + struct task_struct *tsk;
  13898. +
  13899. + /*
  13900. + * sem->readers can only hit 0 when a writer is waiting for the
  13901. + * active readers to leave the critical region.
  13902. + */
  13903. + if (!atomic_dec_and_test(&sem->readers))
  13904. + return;
  13905. +
  13906. + might_sleep();
  13907. + raw_spin_lock_irq(&m->wait_lock);
  13908. + /*
  13909. + * Wake the writer, i.e. the rtmutex owner. It might release the
  13910. + * rtmutex concurrently in the fast path (due to a signal), but to
  13911. + * clean up the rwsem it needs to acquire m->wait_lock. The worst
  13912. + * case which can happen is a spurious wakeup.
  13913. + */
  13914. + tsk = rt_mutex_owner(m);
  13915. + if (tsk)
  13916. + wake_up_process(tsk);
  13917. +
  13918. + raw_spin_unlock_irq(&m->wait_lock);
  13919. +}
  13920. +
  13921. +static void __up_write_unlock(struct rw_semaphore *sem, int bias,
  13922. + unsigned long flags)
  13923. +{
  13924. + struct rt_mutex *m = &sem->rtmutex;
  13925. +
  13926. + atomic_add(READER_BIAS - bias, &sem->readers);
  13927. + raw_spin_unlock_irqrestore(&m->wait_lock, flags);
  13928. + rt_mutex_unlock(m);
  13929. +}
  13930. +
  13931. +static int __sched __down_write_common(struct rw_semaphore *sem, int state)
  13932. +{
  13933. + struct rt_mutex *m = &sem->rtmutex;
  13934. + unsigned long flags;
  13935. +
  13936. + /* Take the rtmutex as a first step */
  13937. + if (rt_mutex_lock_state(m, state))
  13938. + return -EINTR;
  13939. +
  13940. + /* Force readers into slow path */
  13941. + atomic_sub(READER_BIAS, &sem->readers);
  13942. + might_sleep();
  13943. +
  13944. + set_current_state(state);
  13945. + for (;;) {
  13946. + raw_spin_lock_irqsave(&m->wait_lock, flags);
  13947. + /* Have all readers left the critical region? */
  13948. + if (!atomic_read(&sem->readers)) {
  13949. + atomic_set(&sem->readers, WRITER_BIAS);
  13950. + __set_current_state(TASK_RUNNING);
  13951. + raw_spin_unlock_irqrestore(&m->wait_lock, flags);
  13952. + return 0;
  13953. + }
  13954. +
  13955. + if (signal_pending_state(state, current)) {
  13956. + __set_current_state(TASK_RUNNING);
  13957. + __up_write_unlock(sem, 0, flags);
  13958. + return -EINTR;
  13959. + }
  13960. + raw_spin_unlock_irqrestore(&m->wait_lock, flags);
  13961. +
  13962. + if (atomic_read(&sem->readers) != 0) {
  13963. + schedule();
  13964. + set_current_state(state);
  13965. + }
  13966. + }
  13967. +}
  13968. +
  13969. +void __sched __down_write(struct rw_semaphore *sem)
  13970. +{
  13971. + __down_write_common(sem, TASK_UNINTERRUPTIBLE);
  13972. +}
  13973. +
  13974. +int __sched __down_write_killable(struct rw_semaphore *sem)
  13975. +{
  13976. + return __down_write_common(sem, TASK_KILLABLE);
  13977. +}
  13978. +
  13979. +int __down_write_trylock(struct rw_semaphore *sem)
  13980. +{
  13981. + struct rt_mutex *m = &sem->rtmutex;
  13982. + unsigned long flags;
  13983. +
  13984. + if (!rt_mutex_trylock(m))
  13985. + return 0;
  13986. +
  13987. + atomic_sub(READER_BIAS, &sem->readers);
  13988. +
  13989. + raw_spin_lock_irqsave(&m->wait_lock, flags);
  13990. + if (!atomic_read(&sem->readers)) {
  13991. + atomic_set(&sem->readers, WRITER_BIAS);
  13992. + raw_spin_unlock_irqrestore(&m->wait_lock, flags);
  13993. + return 1;
  13994. + }
  13995. + __up_write_unlock(sem, 0, flags);
  13996. + return 0;
  13997. +}
  13998. +
  13999. +void __up_write(struct rw_semaphore *sem)
  14000. +{
  14001. + struct rt_mutex *m = &sem->rtmutex;
  14002. + unsigned long flags;
  14003. +
  14004. + raw_spin_lock_irqsave(&m->wait_lock, flags);
  14005. + __up_write_unlock(sem, WRITER_BIAS, flags);
  14006. +}
  14007. +
  14008. +void __downgrade_write(struct rw_semaphore *sem)
  14009. +{
  14010. + struct rt_mutex *m = &sem->rtmutex;
  14011. + unsigned long flags;
  14012. +
  14013. + raw_spin_lock_irqsave(&m->wait_lock, flags);
  14014. + /* Release it and account current as reader */
  14015. + __up_write_unlock(sem, WRITER_BIAS - 1, flags);
  14016. +}
  14017. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/spinlock.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/spinlock.c
  14018. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/spinlock.c 2017-04-16 10:38:29.000000000 +0200
  14019. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/spinlock.c 2017-04-18 17:54:26.000000000 +0200
  14020. @@ -124,8 +124,11 @@
  14021. * __[spin|read|write]_lock_bh()
  14022. */
  14023. BUILD_LOCK_OPS(spin, raw_spinlock);
  14024. +
  14025. +#ifndef CONFIG_PREEMPT_RT_FULL
  14026. BUILD_LOCK_OPS(read, rwlock);
  14027. BUILD_LOCK_OPS(write, rwlock);
  14028. +#endif
  14029. #endif
  14030. @@ -209,6 +212,8 @@
  14031. EXPORT_SYMBOL(_raw_spin_unlock_bh);
  14032. #endif
  14033. +#ifndef CONFIG_PREEMPT_RT_FULL
  14034. +
  14035. #ifndef CONFIG_INLINE_READ_TRYLOCK
  14036. int __lockfunc _raw_read_trylock(rwlock_t *lock)
  14037. {
  14038. @@ -353,6 +358,8 @@
  14039. EXPORT_SYMBOL(_raw_write_unlock_bh);
  14040. #endif
  14041. +#endif /* !PREEMPT_RT_FULL */
  14042. +
  14043. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  14044. void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass)
  14045. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/spinlock_debug.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/spinlock_debug.c
  14046. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/locking/spinlock_debug.c 2017-04-16 10:38:29.000000000 +0200
  14047. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/locking/spinlock_debug.c 2017-04-18 17:54:26.000000000 +0200
  14048. @@ -31,6 +31,7 @@
  14049. EXPORT_SYMBOL(__raw_spin_lock_init);
  14050. +#ifndef CONFIG_PREEMPT_RT_FULL
  14051. void __rwlock_init(rwlock_t *lock, const char *name,
  14052. struct lock_class_key *key)
  14053. {
  14054. @@ -48,6 +49,7 @@
  14055. }
  14056. EXPORT_SYMBOL(__rwlock_init);
  14057. +#endif
  14058. static void spin_dump(raw_spinlock_t *lock, const char *msg)
  14059. {
  14060. @@ -159,6 +161,7 @@
  14061. arch_spin_unlock(&lock->raw_lock);
  14062. }
  14063. +#ifndef CONFIG_PREEMPT_RT_FULL
  14064. static void rwlock_bug(rwlock_t *lock, const char *msg)
  14065. {
  14066. if (!debug_locks_off())
  14067. @@ -300,3 +303,5 @@
  14068. debug_write_unlock(lock);
  14069. arch_write_unlock(&lock->raw_lock);
  14070. }
  14071. +
  14072. +#endif
  14073. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/module.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/module.c
  14074. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/module.c 2017-04-16 10:38:29.000000000 +0200
  14075. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/module.c 2017-04-18 17:54:26.000000000 +0200
  14076. @@ -660,16 +660,7 @@
  14077. memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
  14078. }
  14079. -/**
  14080. - * is_module_percpu_address - test whether address is from module static percpu
  14081. - * @addr: address to test
  14082. - *
  14083. - * Test whether @addr belongs to module static percpu area.
  14084. - *
  14085. - * RETURNS:
  14086. - * %true if @addr is from module static percpu area
  14087. - */
  14088. -bool is_module_percpu_address(unsigned long addr)
  14089. +bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
  14090. {
  14091. struct module *mod;
  14092. unsigned int cpu;
  14093. @@ -683,9 +674,15 @@
  14094. continue;
  14095. for_each_possible_cpu(cpu) {
  14096. void *start = per_cpu_ptr(mod->percpu, cpu);
  14097. + void *va = (void *)addr;
  14098. - if ((void *)addr >= start &&
  14099. - (void *)addr < start + mod->percpu_size) {
  14100. + if (va >= start && va < start + mod->percpu_size) {
  14101. + if (can_addr) {
  14102. + *can_addr = (unsigned long) (va - start);
  14103. + *can_addr += (unsigned long)
  14104. + per_cpu_ptr(mod->percpu,
  14105. + get_boot_cpu_id());
  14106. + }
  14107. preempt_enable();
  14108. return true;
  14109. }
  14110. @@ -696,6 +693,20 @@
  14111. return false;
  14112. }
  14113. +/**
  14114. + * is_module_percpu_address - test whether address is from module static percpu
  14115. + * @addr: address to test
  14116. + *
  14117. + * Test whether @addr belongs to module static percpu area.
  14118. + *
  14119. + * RETURNS:
  14120. + * %true if @addr is from module static percpu area
  14121. + */
  14122. +bool is_module_percpu_address(unsigned long addr)
  14123. +{
  14124. + return __is_module_percpu_address(addr, NULL);
  14125. +}
  14126. +
  14127. #else /* ... !CONFIG_SMP */
  14128. static inline void __percpu *mod_percpu(struct module *mod)
  14129. @@ -727,6 +738,11 @@
  14130. return false;
  14131. }
  14132. +bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
  14133. +{
  14134. + return false;
  14135. +}
  14136. +
  14137. #endif /* CONFIG_SMP */
  14138. #define MODINFO_ATTR(field) \
  14139. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/panic.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/panic.c
  14140. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/panic.c 2017-04-16 10:38:29.000000000 +0200
  14141. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/panic.c 2017-04-18 17:54:26.000000000 +0200
  14142. @@ -482,9 +482,11 @@
  14143. static int init_oops_id(void)
  14144. {
  14145. +#ifndef CONFIG_PREEMPT_RT_FULL
  14146. if (!oops_id)
  14147. get_random_bytes(&oops_id, sizeof(oops_id));
  14148. else
  14149. +#endif
  14150. oops_id++;
  14151. return 0;
  14152. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/power/hibernate.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/power/hibernate.c
  14153. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/power/hibernate.c 2017-04-16 10:38:29.000000000 +0200
  14154. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/power/hibernate.c 2017-04-18 17:54:26.000000000 +0200
  14155. @@ -286,6 +286,8 @@
  14156. local_irq_disable();
  14157. + system_state = SYSTEM_SUSPEND;
  14158. +
  14159. error = syscore_suspend();
  14160. if (error) {
  14161. printk(KERN_ERR "PM: Some system devices failed to power down, "
  14162. @@ -317,6 +319,7 @@
  14163. syscore_resume();
  14164. Enable_irqs:
  14165. + system_state = SYSTEM_RUNNING;
  14166. local_irq_enable();
  14167. Enable_cpus:
  14168. @@ -446,6 +449,7 @@
  14169. goto Enable_cpus;
  14170. local_irq_disable();
  14171. + system_state = SYSTEM_SUSPEND;
  14172. error = syscore_suspend();
  14173. if (error)
  14174. @@ -479,6 +483,7 @@
  14175. syscore_resume();
  14176. Enable_irqs:
  14177. + system_state = SYSTEM_RUNNING;
  14178. local_irq_enable();
  14179. Enable_cpus:
  14180. @@ -564,6 +569,7 @@
  14181. goto Enable_cpus;
  14182. local_irq_disable();
  14183. + system_state = SYSTEM_SUSPEND;
  14184. syscore_suspend();
  14185. if (pm_wakeup_pending()) {
  14186. error = -EAGAIN;
  14187. @@ -576,6 +582,7 @@
  14188. Power_up:
  14189. syscore_resume();
  14190. + system_state = SYSTEM_RUNNING;
  14191. local_irq_enable();
  14192. Enable_cpus:
  14193. @@ -676,6 +683,10 @@
  14194. return error;
  14195. }
  14196. +#ifndef CONFIG_SUSPEND
  14197. +bool pm_in_action;
  14198. +#endif
  14199. +
  14200. /**
  14201. * hibernate - Carry out system hibernation, including saving the image.
  14202. */
  14203. @@ -689,6 +700,8 @@
  14204. return -EPERM;
  14205. }
  14206. + pm_in_action = true;
  14207. +
  14208. lock_system_sleep();
  14209. /* The snapshot device should not be opened while we're running */
  14210. if (!atomic_add_unless(&snapshot_device_available, -1, 0)) {
  14211. @@ -766,6 +779,7 @@
  14212. atomic_inc(&snapshot_device_available);
  14213. Unlock:
  14214. unlock_system_sleep();
  14215. + pm_in_action = false;
  14216. return error;
  14217. }
  14218. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/power/suspend.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/power/suspend.c
  14219. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/power/suspend.c 2017-04-16 10:38:29.000000000 +0200
  14220. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/power/suspend.c 2017-04-18 17:54:26.000000000 +0200
  14221. @@ -369,6 +369,8 @@
  14222. arch_suspend_disable_irqs();
  14223. BUG_ON(!irqs_disabled());
  14224. + system_state = SYSTEM_SUSPEND;
  14225. +
  14226. error = syscore_suspend();
  14227. if (!error) {
  14228. *wakeup = pm_wakeup_pending();
  14229. @@ -385,6 +387,8 @@
  14230. syscore_resume();
  14231. }
  14232. + system_state = SYSTEM_RUNNING;
  14233. +
  14234. arch_suspend_enable_irqs();
  14235. BUG_ON(irqs_disabled());
  14236. @@ -527,6 +531,8 @@
  14237. return error;
  14238. }
  14239. +bool pm_in_action;
  14240. +
  14241. /**
  14242. * pm_suspend - Externally visible function for suspending the system.
  14243. * @state: System sleep state to enter.
  14244. @@ -541,6 +547,8 @@
  14245. if (state <= PM_SUSPEND_ON || state >= PM_SUSPEND_MAX)
  14246. return -EINVAL;
  14247. + pm_in_action = true;
  14248. +
  14249. error = enter_state(state);
  14250. if (error) {
  14251. suspend_stats.fail++;
  14252. @@ -548,6 +556,7 @@
  14253. } else {
  14254. suspend_stats.success++;
  14255. }
  14256. + pm_in_action = false;
  14257. return error;
  14258. }
  14259. EXPORT_SYMBOL(pm_suspend);
  14260. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/printk/printk.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/printk/printk.c
  14261. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/printk/printk.c 2017-04-16 10:38:29.000000000 +0200
  14262. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/printk/printk.c 2017-04-18 17:54:26.000000000 +0200
  14263. @@ -351,6 +351,65 @@
  14264. */
  14265. DEFINE_RAW_SPINLOCK(logbuf_lock);
  14266. +#ifdef CONFIG_EARLY_PRINTK
  14267. +struct console *early_console;
  14268. +
  14269. +static void early_vprintk(const char *fmt, va_list ap)
  14270. +{
  14271. + if (early_console) {
  14272. + char buf[512];
  14273. + int n = vscnprintf(buf, sizeof(buf), fmt, ap);
  14274. +
  14275. + early_console->write(early_console, buf, n);
  14276. + }
  14277. +}
  14278. +
  14279. +asmlinkage void early_printk(const char *fmt, ...)
  14280. +{
  14281. + va_list ap;
  14282. +
  14283. + va_start(ap, fmt);
  14284. + early_vprintk(fmt, ap);
  14285. + va_end(ap);
  14286. +}
  14287. +
  14288. +/*
  14289. + * This is independent of any log levels - a global
  14290. + * kill switch that turns off all of printk.
  14291. + *
  14292. + * Used by the NMI watchdog if early-printk is enabled.
  14293. + */
  14294. +static bool __read_mostly printk_killswitch;
  14295. +
  14296. +static int __init force_early_printk_setup(char *str)
  14297. +{
  14298. + printk_killswitch = true;
  14299. + return 0;
  14300. +}
  14301. +early_param("force_early_printk", force_early_printk_setup);
  14302. +
  14303. +void printk_kill(void)
  14304. +{
  14305. + printk_killswitch = true;
  14306. +}
  14307. +
  14308. +#ifdef CONFIG_PRINTK
  14309. +static int forced_early_printk(const char *fmt, va_list ap)
  14310. +{
  14311. + if (!printk_killswitch)
  14312. + return 0;
  14313. + early_vprintk(fmt, ap);
  14314. + return 1;
  14315. +}
  14316. +#endif
  14317. +
  14318. +#else
  14319. +static inline int forced_early_printk(const char *fmt, va_list ap)
  14320. +{
  14321. + return 0;
  14322. +}
  14323. +#endif
  14324. +
  14325. #ifdef CONFIG_PRINTK
  14326. DECLARE_WAIT_QUEUE_HEAD(log_wait);
  14327. /* the next printk record to read by syslog(READ) or /proc/kmsg */
  14328. @@ -1337,6 +1396,7 @@
  14329. {
  14330. char *text;
  14331. int len = 0;
  14332. + int attempts = 0;
  14333. text = kmalloc(LOG_LINE_MAX + PREFIX_MAX, GFP_KERNEL);
  14334. if (!text)
  14335. @@ -1348,6 +1408,14 @@
  14336. u64 seq;
  14337. u32 idx;
  14338. enum log_flags prev;
  14339. + int num_msg;
  14340. +try_again:
  14341. + attempts++;
  14342. + if (attempts > 10) {
  14343. + len = -EBUSY;
  14344. + goto out;
  14345. + }
  14346. + num_msg = 0;
  14347. /*
  14348. * Find first record that fits, including all following records,
  14349. @@ -1363,6 +1431,14 @@
  14350. prev = msg->flags;
  14351. idx = log_next(idx);
  14352. seq++;
  14353. + num_msg++;
  14354. + if (num_msg > 5) {
  14355. + num_msg = 0;
  14356. + raw_spin_unlock_irq(&logbuf_lock);
  14357. + raw_spin_lock_irq(&logbuf_lock);
  14358. + if (clear_seq < log_first_seq)
  14359. + goto try_again;
  14360. + }
  14361. }
  14362. /* move first record forward until length fits into the buffer */
  14363. @@ -1376,6 +1452,14 @@
  14364. prev = msg->flags;
  14365. idx = log_next(idx);
  14366. seq++;
  14367. + num_msg++;
  14368. + if (num_msg > 5) {
  14369. + num_msg = 0;
  14370. + raw_spin_unlock_irq(&logbuf_lock);
  14371. + raw_spin_lock_irq(&logbuf_lock);
  14372. + if (clear_seq < log_first_seq)
  14373. + goto try_again;
  14374. + }
  14375. }
  14376. /* last message fitting into this dump */
  14377. @@ -1416,6 +1500,7 @@
  14378. clear_seq = log_next_seq;
  14379. clear_idx = log_next_idx;
  14380. }
  14381. +out:
  14382. raw_spin_unlock_irq(&logbuf_lock);
  14383. kfree(text);
  14384. @@ -1569,6 +1654,12 @@
  14385. if (!console_drivers)
  14386. return;
  14387. + if (IS_ENABLED(CONFIG_PREEMPT_RT_BASE)) {
  14388. + if (in_irq() || in_nmi())
  14389. + return;
  14390. + }
  14391. +
  14392. + migrate_disable();
  14393. for_each_console(con) {
  14394. if (exclusive_console && con != exclusive_console)
  14395. continue;
  14396. @@ -1584,6 +1675,7 @@
  14397. else
  14398. con->write(con, text, len);
  14399. }
  14400. + migrate_enable();
  14401. }
  14402. /*
  14403. @@ -1781,6 +1873,13 @@
  14404. /* cpu currently holding logbuf_lock in this function */
  14405. static unsigned int logbuf_cpu = UINT_MAX;
  14406. + /*
  14407. + * Fall back to early_printk if a debugging subsystem has
  14408. + * killed printk output
  14409. + */
  14410. + if (unlikely(forced_early_printk(fmt, args)))
  14411. + return 1;
  14412. +
  14413. if (level == LOGLEVEL_SCHED) {
  14414. level = LOGLEVEL_DEFAULT;
  14415. in_sched = true;
  14416. @@ -1885,13 +1984,23 @@
  14417. /* If called from the scheduler, we can not call up(). */
  14418. if (!in_sched) {
  14419. + int may_trylock = 1;
  14420. +
  14421. lockdep_off();
  14422. +#ifdef CONFIG_PREEMPT_RT_FULL
  14423. + /*
  14424. + * we can't take a sleeping lock with IRQs or preeption disabled
  14425. + * so we can't print in these contexts
  14426. + */
  14427. + if (!(preempt_count() == 0 && !irqs_disabled()))
  14428. + may_trylock = 0;
  14429. +#endif
  14430. /*
  14431. * Try to acquire and then immediately release the console
  14432. * semaphore. The release will print out buffers and wake up
  14433. * /dev/kmsg and syslog() users.
  14434. */
  14435. - if (console_trylock())
  14436. + if (may_trylock && console_trylock())
  14437. console_unlock();
  14438. lockdep_on();
  14439. }
  14440. @@ -2014,26 +2123,6 @@
  14441. #endif /* CONFIG_PRINTK */
  14442. -#ifdef CONFIG_EARLY_PRINTK
  14443. -struct console *early_console;
  14444. -
  14445. -asmlinkage __visible void early_printk(const char *fmt, ...)
  14446. -{
  14447. - va_list ap;
  14448. - char buf[512];
  14449. - int n;
  14450. -
  14451. - if (!early_console)
  14452. - return;
  14453. -
  14454. - va_start(ap, fmt);
  14455. - n = vscnprintf(buf, sizeof(buf), fmt, ap);
  14456. - va_end(ap);
  14457. -
  14458. - early_console->write(early_console, buf, n);
  14459. -}
  14460. -#endif
  14461. -
  14462. static int __add_preferred_console(char *name, int idx, char *options,
  14463. char *brl_options)
  14464. {
  14465. @@ -2303,11 +2392,16 @@
  14466. goto out;
  14467. len = cont_print_text(text, size);
  14468. +#ifdef CONFIG_PREEMPT_RT_FULL
  14469. + raw_spin_unlock_irqrestore(&logbuf_lock, flags);
  14470. + call_console_drivers(cont.level, NULL, 0, text, len);
  14471. +#else
  14472. raw_spin_unlock(&logbuf_lock);
  14473. stop_critical_timings();
  14474. call_console_drivers(cont.level, NULL, 0, text, len);
  14475. start_critical_timings();
  14476. local_irq_restore(flags);
  14477. +#endif
  14478. return;
  14479. out:
  14480. raw_spin_unlock_irqrestore(&logbuf_lock, flags);
  14481. @@ -2431,13 +2525,17 @@
  14482. console_idx = log_next(console_idx);
  14483. console_seq++;
  14484. console_prev = msg->flags;
  14485. +#ifdef CONFIG_PREEMPT_RT_FULL
  14486. + raw_spin_unlock_irqrestore(&logbuf_lock, flags);
  14487. + call_console_drivers(level, ext_text, ext_len, text, len);
  14488. +#else
  14489. raw_spin_unlock(&logbuf_lock);
  14490. stop_critical_timings(); /* don't trace print latency */
  14491. call_console_drivers(level, ext_text, ext_len, text, len);
  14492. start_critical_timings();
  14493. local_irq_restore(flags);
  14494. -
  14495. +#endif
  14496. if (do_cond_resched)
  14497. cond_resched();
  14498. }
  14499. @@ -2489,6 +2587,11 @@
  14500. {
  14501. struct console *c;
  14502. + if (IS_ENABLED(CONFIG_PREEMPT_RT_BASE)) {
  14503. + if (in_irq() || in_nmi())
  14504. + return;
  14505. + }
  14506. +
  14507. /*
  14508. * console_unblank can no longer be called in interrupt context unless
  14509. * oops_in_progress is set to 1..
  14510. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/ptrace.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/ptrace.c
  14511. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/ptrace.c 2017-04-16 10:38:29.000000000 +0200
  14512. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/ptrace.c 2017-04-18 17:54:26.000000000 +0200
  14513. @@ -166,7 +166,14 @@
  14514. spin_lock_irq(&task->sighand->siglock);
  14515. if (task_is_traced(task) && !__fatal_signal_pending(task)) {
  14516. - task->state = __TASK_TRACED;
  14517. + unsigned long flags;
  14518. +
  14519. + raw_spin_lock_irqsave(&task->pi_lock, flags);
  14520. + if (task->state & __TASK_TRACED)
  14521. + task->state = __TASK_TRACED;
  14522. + else
  14523. + task->saved_state = __TASK_TRACED;
  14524. + raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  14525. ret = true;
  14526. }
  14527. spin_unlock_irq(&task->sighand->siglock);
  14528. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/rcu/rcutorture.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/rcu/rcutorture.c
  14529. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/rcu/rcutorture.c 2017-04-16 10:38:29.000000000 +0200
  14530. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/rcu/rcutorture.c 2017-04-18 17:54:26.000000000 +0200
  14531. @@ -404,6 +404,7 @@
  14532. .name = "rcu"
  14533. };
  14534. +#ifndef CONFIG_PREEMPT_RT_FULL
  14535. /*
  14536. * Definitions for rcu_bh torture testing.
  14537. */
  14538. @@ -443,6 +444,12 @@
  14539. .name = "rcu_bh"
  14540. };
  14541. +#else
  14542. +static struct rcu_torture_ops rcu_bh_ops = {
  14543. + .ttype = INVALID_RCU_FLAVOR,
  14544. +};
  14545. +#endif
  14546. +
  14547. /*
  14548. * Don't even think about trying any of these in real life!!!
  14549. * The names includes "busted", and they really means it!
  14550. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/rcu/tree.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/rcu/tree.c
  14551. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/rcu/tree.c 2017-04-16 10:38:29.000000000 +0200
  14552. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/rcu/tree.c 2017-04-18 17:54:26.000000000 +0200
  14553. @@ -55,6 +55,11 @@
  14554. #include <linux/random.h>
  14555. #include <linux/trace_events.h>
  14556. #include <linux/suspend.h>
  14557. +#include <linux/delay.h>
  14558. +#include <linux/gfp.h>
  14559. +#include <linux/oom.h>
  14560. +#include <linux/smpboot.h>
  14561. +#include "../time/tick-internal.h"
  14562. #include "tree.h"
  14563. #include "rcu.h"
  14564. @@ -260,6 +265,19 @@
  14565. this_cpu_ptr(&rcu_sched_data), true);
  14566. }
  14567. +#ifdef CONFIG_PREEMPT_RT_FULL
  14568. +static void rcu_preempt_qs(void);
  14569. +
  14570. +void rcu_bh_qs(void)
  14571. +{
  14572. + unsigned long flags;
  14573. +
  14574. + /* Callers to this function, rcu_preempt_qs(), must disable irqs. */
  14575. + local_irq_save(flags);
  14576. + rcu_preempt_qs();
  14577. + local_irq_restore(flags);
  14578. +}
  14579. +#else
  14580. void rcu_bh_qs(void)
  14581. {
  14582. if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) {
  14583. @@ -269,6 +287,7 @@
  14584. __this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);
  14585. }
  14586. }
  14587. +#endif
  14588. static DEFINE_PER_CPU(int, rcu_sched_qs_mask);
  14589. @@ -449,11 +468,13 @@
  14590. /*
  14591. * Return the number of RCU BH batches started thus far for debug & stats.
  14592. */
  14593. +#ifndef CONFIG_PREEMPT_RT_FULL
  14594. unsigned long rcu_batches_started_bh(void)
  14595. {
  14596. return rcu_bh_state.gpnum;
  14597. }
  14598. EXPORT_SYMBOL_GPL(rcu_batches_started_bh);
  14599. +#endif
  14600. /*
  14601. * Return the number of RCU batches completed thus far for debug & stats.
  14602. @@ -473,6 +494,7 @@
  14603. }
  14604. EXPORT_SYMBOL_GPL(rcu_batches_completed_sched);
  14605. +#ifndef CONFIG_PREEMPT_RT_FULL
  14606. /*
  14607. * Return the number of RCU BH batches completed thus far for debug & stats.
  14608. */
  14609. @@ -481,6 +503,7 @@
  14610. return rcu_bh_state.completed;
  14611. }
  14612. EXPORT_SYMBOL_GPL(rcu_batches_completed_bh);
  14613. +#endif
  14614. /*
  14615. * Return the number of RCU expedited batches completed thus far for
  14616. @@ -504,6 +527,7 @@
  14617. }
  14618. EXPORT_SYMBOL_GPL(rcu_exp_batches_completed_sched);
  14619. +#ifndef CONFIG_PREEMPT_RT_FULL
  14620. /*
  14621. * Force a quiescent state.
  14622. */
  14623. @@ -522,6 +546,13 @@
  14624. }
  14625. EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state);
  14626. +#else
  14627. +void rcu_force_quiescent_state(void)
  14628. +{
  14629. +}
  14630. +EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
  14631. +#endif
  14632. +
  14633. /*
  14634. * Force a quiescent state for RCU-sched.
  14635. */
  14636. @@ -572,9 +603,11 @@
  14637. case RCU_FLAVOR:
  14638. rsp = rcu_state_p;
  14639. break;
  14640. +#ifndef CONFIG_PREEMPT_RT_FULL
  14641. case RCU_BH_FLAVOR:
  14642. rsp = &rcu_bh_state;
  14643. break;
  14644. +#endif
  14645. case RCU_SCHED_FLAVOR:
  14646. rsp = &rcu_sched_state;
  14647. break;
  14648. @@ -3016,18 +3049,17 @@
  14649. /*
  14650. * Do RCU core processing for the current CPU.
  14651. */
  14652. -static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused)
  14653. +static __latent_entropy void rcu_process_callbacks(void)
  14654. {
  14655. struct rcu_state *rsp;
  14656. if (cpu_is_offline(smp_processor_id()))
  14657. return;
  14658. - trace_rcu_utilization(TPS("Start RCU core"));
  14659. for_each_rcu_flavor(rsp)
  14660. __rcu_process_callbacks(rsp);
  14661. - trace_rcu_utilization(TPS("End RCU core"));
  14662. }
  14663. +static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
  14664. /*
  14665. * Schedule RCU callback invocation. If the specified type of RCU
  14666. * does not support RCU priority boosting, just do a direct call,
  14667. @@ -3039,18 +3071,105 @@
  14668. {
  14669. if (unlikely(!READ_ONCE(rcu_scheduler_fully_active)))
  14670. return;
  14671. - if (likely(!rsp->boost)) {
  14672. - rcu_do_batch(rsp, rdp);
  14673. + rcu_do_batch(rsp, rdp);
  14674. +}
  14675. +
  14676. +static void rcu_wake_cond(struct task_struct *t, int status)
  14677. +{
  14678. + /*
  14679. + * If the thread is yielding, only wake it when this
  14680. + * is invoked from idle
  14681. + */
  14682. + if (t && (status != RCU_KTHREAD_YIELDING || is_idle_task(current)))
  14683. + wake_up_process(t);
  14684. +}
  14685. +
  14686. +/*
  14687. + * Wake up this CPU's rcuc kthread to do RCU core processing.
  14688. + */
  14689. +static void invoke_rcu_core(void)
  14690. +{
  14691. + unsigned long flags;
  14692. + struct task_struct *t;
  14693. +
  14694. + if (!cpu_online(smp_processor_id()))
  14695. return;
  14696. + local_irq_save(flags);
  14697. + __this_cpu_write(rcu_cpu_has_work, 1);
  14698. + t = __this_cpu_read(rcu_cpu_kthread_task);
  14699. + if (t != NULL && current != t)
  14700. + rcu_wake_cond(t, __this_cpu_read(rcu_cpu_kthread_status));
  14701. + local_irq_restore(flags);
  14702. +}
  14703. +
  14704. +static void rcu_cpu_kthread_park(unsigned int cpu)
  14705. +{
  14706. + per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
  14707. +}
  14708. +
  14709. +static int rcu_cpu_kthread_should_run(unsigned int cpu)
  14710. +{
  14711. + return __this_cpu_read(rcu_cpu_has_work);
  14712. +}
  14713. +
  14714. +/*
  14715. + * Per-CPU kernel thread that invokes RCU callbacks. This replaces the
  14716. + * RCU softirq used in flavors and configurations of RCU that do not
  14717. + * support RCU priority boosting.
  14718. + */
  14719. +static void rcu_cpu_kthread(unsigned int cpu)
  14720. +{
  14721. + unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
  14722. + char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
  14723. + int spincnt;
  14724. +
  14725. + for (spincnt = 0; spincnt < 10; spincnt++) {
  14726. + trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
  14727. + local_bh_disable();
  14728. + *statusp = RCU_KTHREAD_RUNNING;
  14729. + this_cpu_inc(rcu_cpu_kthread_loops);
  14730. + local_irq_disable();
  14731. + work = *workp;
  14732. + *workp = 0;
  14733. + local_irq_enable();
  14734. + if (work)
  14735. + rcu_process_callbacks();
  14736. + local_bh_enable();
  14737. + if (*workp == 0) {
  14738. + trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
  14739. + *statusp = RCU_KTHREAD_WAITING;
  14740. + return;
  14741. + }
  14742. }
  14743. - invoke_rcu_callbacks_kthread();
  14744. + *statusp = RCU_KTHREAD_YIELDING;
  14745. + trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
  14746. + schedule_timeout_interruptible(2);
  14747. + trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
  14748. + *statusp = RCU_KTHREAD_WAITING;
  14749. }
  14750. -static void invoke_rcu_core(void)
  14751. +static struct smp_hotplug_thread rcu_cpu_thread_spec = {
  14752. + .store = &rcu_cpu_kthread_task,
  14753. + .thread_should_run = rcu_cpu_kthread_should_run,
  14754. + .thread_fn = rcu_cpu_kthread,
  14755. + .thread_comm = "rcuc/%u",
  14756. + .setup = rcu_cpu_kthread_setup,
  14757. + .park = rcu_cpu_kthread_park,
  14758. +};
  14759. +
  14760. +/*
  14761. + * Spawn per-CPU RCU core processing kthreads.
  14762. + */
  14763. +static int __init rcu_spawn_core_kthreads(void)
  14764. {
  14765. - if (cpu_online(smp_processor_id()))
  14766. - raise_softirq(RCU_SOFTIRQ);
  14767. + int cpu;
  14768. +
  14769. + for_each_possible_cpu(cpu)
  14770. + per_cpu(rcu_cpu_has_work, cpu) = 0;
  14771. + BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
  14772. + return 0;
  14773. }
  14774. +early_initcall(rcu_spawn_core_kthreads);
  14775. /*
  14776. * Handle any core-RCU processing required by a call_rcu() invocation.
  14777. @@ -3195,6 +3314,7 @@
  14778. }
  14779. EXPORT_SYMBOL_GPL(call_rcu_sched);
  14780. +#ifndef CONFIG_PREEMPT_RT_FULL
  14781. /*
  14782. * Queue an RCU callback for invocation after a quicker grace period.
  14783. */
  14784. @@ -3203,6 +3323,7 @@
  14785. __call_rcu(head, func, &rcu_bh_state, -1, 0);
  14786. }
  14787. EXPORT_SYMBOL_GPL(call_rcu_bh);
  14788. +#endif
  14789. /*
  14790. * Queue an RCU callback for lazy invocation after a grace period.
  14791. @@ -3294,6 +3415,7 @@
  14792. }
  14793. EXPORT_SYMBOL_GPL(synchronize_sched);
  14794. +#ifndef CONFIG_PREEMPT_RT_FULL
  14795. /**
  14796. * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed.
  14797. *
  14798. @@ -3320,6 +3442,7 @@
  14799. wait_rcu_gp(call_rcu_bh);
  14800. }
  14801. EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
  14802. +#endif
  14803. /**
  14804. * get_state_synchronize_rcu - Snapshot current RCU state
  14805. @@ -3698,6 +3821,7 @@
  14806. mutex_unlock(&rsp->barrier_mutex);
  14807. }
  14808. +#ifndef CONFIG_PREEMPT_RT_FULL
  14809. /**
  14810. * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete.
  14811. */
  14812. @@ -3706,6 +3830,7 @@
  14813. _rcu_barrier(&rcu_bh_state);
  14814. }
  14815. EXPORT_SYMBOL_GPL(rcu_barrier_bh);
  14816. +#endif
  14817. /**
  14818. * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks.
  14819. @@ -4227,12 +4352,13 @@
  14820. rcu_bootup_announce();
  14821. rcu_init_geometry();
  14822. +#ifndef CONFIG_PREEMPT_RT_FULL
  14823. rcu_init_one(&rcu_bh_state);
  14824. +#endif
  14825. rcu_init_one(&rcu_sched_state);
  14826. if (dump_tree)
  14827. rcu_dump_rcu_node_tree(&rcu_sched_state);
  14828. __rcu_init_preempt();
  14829. - open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
  14830. /*
  14831. * We don't need protection against CPU-hotplug here because
  14832. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/rcu/tree.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/rcu/tree.h
  14833. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/rcu/tree.h 2017-04-16 10:38:29.000000000 +0200
  14834. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/rcu/tree.h 2017-04-18 17:54:26.000000000 +0200
  14835. @@ -588,18 +588,18 @@
  14836. */
  14837. extern struct rcu_state rcu_sched_state;
  14838. +#ifndef CONFIG_PREEMPT_RT_FULL
  14839. extern struct rcu_state rcu_bh_state;
  14840. +#endif
  14841. #ifdef CONFIG_PREEMPT_RCU
  14842. extern struct rcu_state rcu_preempt_state;
  14843. #endif /* #ifdef CONFIG_PREEMPT_RCU */
  14844. -#ifdef CONFIG_RCU_BOOST
  14845. DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
  14846. DECLARE_PER_CPU(int, rcu_cpu_kthread_cpu);
  14847. DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
  14848. DECLARE_PER_CPU(char, rcu_cpu_has_work);
  14849. -#endif /* #ifdef CONFIG_RCU_BOOST */
  14850. #ifndef RCU_TREE_NONCORE
  14851. @@ -619,10 +619,9 @@
  14852. static void __init __rcu_init_preempt(void);
  14853. static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);
  14854. static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);
  14855. -static void invoke_rcu_callbacks_kthread(void);
  14856. static bool rcu_is_callbacks_kthread(void);
  14857. +static void rcu_cpu_kthread_setup(unsigned int cpu);
  14858. #ifdef CONFIG_RCU_BOOST
  14859. -static void rcu_preempt_do_callbacks(void);
  14860. static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
  14861. struct rcu_node *rnp);
  14862. #endif /* #ifdef CONFIG_RCU_BOOST */
  14863. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/rcu/tree_plugin.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/rcu/tree_plugin.h
  14864. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/rcu/tree_plugin.h 2017-04-16 10:38:29.000000000 +0200
  14865. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/rcu/tree_plugin.h 2017-04-18 17:54:26.000000000 +0200
  14866. @@ -24,25 +24,10 @@
  14867. * Paul E. McKenney <paulmck@linux.vnet.ibm.com>
  14868. */
  14869. -#include <linux/delay.h>
  14870. -#include <linux/gfp.h>
  14871. -#include <linux/oom.h>
  14872. -#include <linux/smpboot.h>
  14873. -#include "../time/tick-internal.h"
  14874. -
  14875. #ifdef CONFIG_RCU_BOOST
  14876. #include "../locking/rtmutex_common.h"
  14877. -/*
  14878. - * Control variables for per-CPU and per-rcu_node kthreads. These
  14879. - * handle all flavors of RCU.
  14880. - */
  14881. -static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
  14882. -DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
  14883. -DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
  14884. -DEFINE_PER_CPU(char, rcu_cpu_has_work);
  14885. -
  14886. #else /* #ifdef CONFIG_RCU_BOOST */
  14887. /*
  14888. @@ -55,6 +40,14 @@
  14889. #endif /* #else #ifdef CONFIG_RCU_BOOST */
  14890. +/*
  14891. + * Control variables for per-CPU and per-rcu_node kthreads. These
  14892. + * handle all flavors of RCU.
  14893. + */
  14894. +DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
  14895. +DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
  14896. +DEFINE_PER_CPU(char, rcu_cpu_has_work);
  14897. +
  14898. #ifdef CONFIG_RCU_NOCB_CPU
  14899. static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
  14900. static bool have_rcu_nocb_mask; /* Was rcu_nocb_mask allocated? */
  14901. @@ -426,7 +419,7 @@
  14902. }
  14903. /* Hardware IRQ handlers cannot block, complain if they get here. */
  14904. - if (in_irq() || in_serving_softirq()) {
  14905. + if (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_OFFSET)) {
  14906. lockdep_rcu_suspicious(__FILE__, __LINE__,
  14907. "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n");
  14908. pr_alert("->rcu_read_unlock_special: %#x (b: %d, enq: %d nq: %d)\n",
  14909. @@ -632,15 +625,6 @@
  14910. t->rcu_read_unlock_special.b.need_qs = true;
  14911. }
  14912. -#ifdef CONFIG_RCU_BOOST
  14913. -
  14914. -static void rcu_preempt_do_callbacks(void)
  14915. -{
  14916. - rcu_do_batch(rcu_state_p, this_cpu_ptr(rcu_data_p));
  14917. -}
  14918. -
  14919. -#endif /* #ifdef CONFIG_RCU_BOOST */
  14920. -
  14921. /*
  14922. * Queue a preemptible-RCU callback for invocation after a grace period.
  14923. */
  14924. @@ -829,6 +813,19 @@
  14925. #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
  14926. +/*
  14927. + * If boosting, set rcuc kthreads to realtime priority.
  14928. + */
  14929. +static void rcu_cpu_kthread_setup(unsigned int cpu)
  14930. +{
  14931. +#ifdef CONFIG_RCU_BOOST
  14932. + struct sched_param sp;
  14933. +
  14934. + sp.sched_priority = kthread_prio;
  14935. + sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
  14936. +#endif /* #ifdef CONFIG_RCU_BOOST */
  14937. +}
  14938. +
  14939. #ifdef CONFIG_RCU_BOOST
  14940. #include "../locking/rtmutex_common.h"
  14941. @@ -860,16 +857,6 @@
  14942. #endif /* #else #ifdef CONFIG_RCU_TRACE */
  14943. -static void rcu_wake_cond(struct task_struct *t, int status)
  14944. -{
  14945. - /*
  14946. - * If the thread is yielding, only wake it when this
  14947. - * is invoked from idle
  14948. - */
  14949. - if (status != RCU_KTHREAD_YIELDING || is_idle_task(current))
  14950. - wake_up_process(t);
  14951. -}
  14952. -
  14953. /*
  14954. * Carry out RCU priority boosting on the task indicated by ->exp_tasks
  14955. * or ->boost_tasks, advancing the pointer to the next task in the
  14956. @@ -1013,23 +1000,6 @@
  14957. }
  14958. /*
  14959. - * Wake up the per-CPU kthread to invoke RCU callbacks.
  14960. - */
  14961. -static void invoke_rcu_callbacks_kthread(void)
  14962. -{
  14963. - unsigned long flags;
  14964. -
  14965. - local_irq_save(flags);
  14966. - __this_cpu_write(rcu_cpu_has_work, 1);
  14967. - if (__this_cpu_read(rcu_cpu_kthread_task) != NULL &&
  14968. - current != __this_cpu_read(rcu_cpu_kthread_task)) {
  14969. - rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task),
  14970. - __this_cpu_read(rcu_cpu_kthread_status));
  14971. - }
  14972. - local_irq_restore(flags);
  14973. -}
  14974. -
  14975. -/*
  14976. * Is the current CPU running the RCU-callbacks kthread?
  14977. * Caller must have preemption disabled.
  14978. */
  14979. @@ -1083,67 +1053,6 @@
  14980. return 0;
  14981. }
  14982. -static void rcu_kthread_do_work(void)
  14983. -{
  14984. - rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data));
  14985. - rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data));
  14986. - rcu_preempt_do_callbacks();
  14987. -}
  14988. -
  14989. -static void rcu_cpu_kthread_setup(unsigned int cpu)
  14990. -{
  14991. - struct sched_param sp;
  14992. -
  14993. - sp.sched_priority = kthread_prio;
  14994. - sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
  14995. -}
  14996. -
  14997. -static void rcu_cpu_kthread_park(unsigned int cpu)
  14998. -{
  14999. - per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
  15000. -}
  15001. -
  15002. -static int rcu_cpu_kthread_should_run(unsigned int cpu)
  15003. -{
  15004. - return __this_cpu_read(rcu_cpu_has_work);
  15005. -}
  15006. -
  15007. -/*
  15008. - * Per-CPU kernel thread that invokes RCU callbacks. This replaces the
  15009. - * RCU softirq used in flavors and configurations of RCU that do not
  15010. - * support RCU priority boosting.
  15011. - */
  15012. -static void rcu_cpu_kthread(unsigned int cpu)
  15013. -{
  15014. - unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
  15015. - char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
  15016. - int spincnt;
  15017. -
  15018. - for (spincnt = 0; spincnt < 10; spincnt++) {
  15019. - trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
  15020. - local_bh_disable();
  15021. - *statusp = RCU_KTHREAD_RUNNING;
  15022. - this_cpu_inc(rcu_cpu_kthread_loops);
  15023. - local_irq_disable();
  15024. - work = *workp;
  15025. - *workp = 0;
  15026. - local_irq_enable();
  15027. - if (work)
  15028. - rcu_kthread_do_work();
  15029. - local_bh_enable();
  15030. - if (*workp == 0) {
  15031. - trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
  15032. - *statusp = RCU_KTHREAD_WAITING;
  15033. - return;
  15034. - }
  15035. - }
  15036. - *statusp = RCU_KTHREAD_YIELDING;
  15037. - trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
  15038. - schedule_timeout_interruptible(2);
  15039. - trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
  15040. - *statusp = RCU_KTHREAD_WAITING;
  15041. -}
  15042. -
  15043. /*
  15044. * Set the per-rcu_node kthread's affinity to cover all CPUs that are
  15045. * served by the rcu_node in question. The CPU hotplug lock is still
  15046. @@ -1174,26 +1083,12 @@
  15047. free_cpumask_var(cm);
  15048. }
  15049. -static struct smp_hotplug_thread rcu_cpu_thread_spec = {
  15050. - .store = &rcu_cpu_kthread_task,
  15051. - .thread_should_run = rcu_cpu_kthread_should_run,
  15052. - .thread_fn = rcu_cpu_kthread,
  15053. - .thread_comm = "rcuc/%u",
  15054. - .setup = rcu_cpu_kthread_setup,
  15055. - .park = rcu_cpu_kthread_park,
  15056. -};
  15057. -
  15058. /*
  15059. * Spawn boost kthreads -- called as soon as the scheduler is running.
  15060. */
  15061. static void __init rcu_spawn_boost_kthreads(void)
  15062. {
  15063. struct rcu_node *rnp;
  15064. - int cpu;
  15065. -
  15066. - for_each_possible_cpu(cpu)
  15067. - per_cpu(rcu_cpu_has_work, cpu) = 0;
  15068. - BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
  15069. rcu_for_each_leaf_node(rcu_state_p, rnp)
  15070. (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
  15071. }
  15072. @@ -1216,11 +1111,6 @@
  15073. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  15074. }
  15075. -static void invoke_rcu_callbacks_kthread(void)
  15076. -{
  15077. - WARN_ON_ONCE(1);
  15078. -}
  15079. -
  15080. static bool rcu_is_callbacks_kthread(void)
  15081. {
  15082. return false;
  15083. @@ -1244,7 +1134,7 @@
  15084. #endif /* #else #ifdef CONFIG_RCU_BOOST */
  15085. -#if !defined(CONFIG_RCU_FAST_NO_HZ)
  15086. +#if !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL)
  15087. /*
  15088. * Check to see if any future RCU-related work will need to be done
  15089. @@ -1261,7 +1151,9 @@
  15090. return IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL)
  15091. ? 0 : rcu_cpu_has_callbacks(NULL);
  15092. }
  15093. +#endif /* !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL) */
  15094. +#if !defined(CONFIG_RCU_FAST_NO_HZ)
  15095. /*
  15096. * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
  15097. * after it.
  15098. @@ -1357,6 +1249,8 @@
  15099. return cbs_ready;
  15100. }
  15101. +#ifndef CONFIG_PREEMPT_RT_FULL
  15102. +
  15103. /*
  15104. * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready
  15105. * to invoke. If the CPU has callbacks, try to advance them. Tell the
  15106. @@ -1402,6 +1296,7 @@
  15107. *nextevt = basemono + dj * TICK_NSEC;
  15108. return 0;
  15109. }
  15110. +#endif /* #ifndef CONFIG_PREEMPT_RT_FULL */
  15111. /*
  15112. * Prepare a CPU for idle from an RCU perspective. The first major task
  15113. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/rcu/update.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/rcu/update.c
  15114. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/rcu/update.c 2017-04-16 10:38:29.000000000 +0200
  15115. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/rcu/update.c 2017-04-18 17:54:26.000000000 +0200
  15116. @@ -62,7 +62,7 @@
  15117. #ifndef CONFIG_TINY_RCU
  15118. module_param(rcu_expedited, int, 0);
  15119. module_param(rcu_normal, int, 0);
  15120. -static int rcu_normal_after_boot;
  15121. +static int rcu_normal_after_boot = IS_ENABLED(CONFIG_PREEMPT_RT_FULL);
  15122. module_param(rcu_normal_after_boot, int, 0);
  15123. #endif /* #ifndef CONFIG_TINY_RCU */
  15124. @@ -132,8 +132,7 @@
  15125. }
  15126. EXPORT_SYMBOL_GPL(rcu_gp_is_normal);
  15127. -static atomic_t rcu_expedited_nesting =
  15128. - ATOMIC_INIT(IS_ENABLED(CONFIG_RCU_EXPEDITE_BOOT) ? 1 : 0);
  15129. +static atomic_t rcu_expedited_nesting = ATOMIC_INIT(1);
  15130. /*
  15131. * Should normal grace-period primitives be expedited? Intended for
  15132. @@ -182,8 +181,7 @@
  15133. */
  15134. void rcu_end_inkernel_boot(void)
  15135. {
  15136. - if (IS_ENABLED(CONFIG_RCU_EXPEDITE_BOOT))
  15137. - rcu_unexpedite_gp();
  15138. + rcu_unexpedite_gp();
  15139. if (rcu_normal_after_boot)
  15140. WRITE_ONCE(rcu_normal, 1);
  15141. }
  15142. @@ -298,6 +296,7 @@
  15143. }
  15144. EXPORT_SYMBOL_GPL(rcu_read_lock_held);
  15145. +#ifndef CONFIG_PREEMPT_RT_FULL
  15146. /**
  15147. * rcu_read_lock_bh_held() - might we be in RCU-bh read-side critical section?
  15148. *
  15149. @@ -324,6 +323,7 @@
  15150. return in_softirq() || irqs_disabled();
  15151. }
  15152. EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
  15153. +#endif
  15154. #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
  15155. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/Makefile linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/Makefile
  15156. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/Makefile 2017-04-16 10:38:29.000000000 +0200
  15157. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/Makefile 2017-04-18 17:54:26.000000000 +0200
  15158. @@ -17,7 +17,7 @@
  15159. obj-y += core.o loadavg.o clock.o cputime.o
  15160. obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o
  15161. -obj-y += wait.o swait.o completion.o idle.o
  15162. +obj-y += wait.o swait.o swork.o completion.o idle.o
  15163. obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o
  15164. obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o
  15165. obj-$(CONFIG_SCHEDSTATS) += stats.o
  15166. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/completion.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/completion.c
  15167. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/completion.c 2017-04-16 10:38:29.000000000 +0200
  15168. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/completion.c 2017-04-18 17:54:26.000000000 +0200
  15169. @@ -30,10 +30,10 @@
  15170. {
  15171. unsigned long flags;
  15172. - spin_lock_irqsave(&x->wait.lock, flags);
  15173. + raw_spin_lock_irqsave(&x->wait.lock, flags);
  15174. x->done++;
  15175. - __wake_up_locked(&x->wait, TASK_NORMAL, 1);
  15176. - spin_unlock_irqrestore(&x->wait.lock, flags);
  15177. + swake_up_locked(&x->wait);
  15178. + raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  15179. }
  15180. EXPORT_SYMBOL(complete);
  15181. @@ -50,10 +50,10 @@
  15182. {
  15183. unsigned long flags;
  15184. - spin_lock_irqsave(&x->wait.lock, flags);
  15185. + raw_spin_lock_irqsave(&x->wait.lock, flags);
  15186. x->done += UINT_MAX/2;
  15187. - __wake_up_locked(&x->wait, TASK_NORMAL, 0);
  15188. - spin_unlock_irqrestore(&x->wait.lock, flags);
  15189. + swake_up_all_locked(&x->wait);
  15190. + raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  15191. }
  15192. EXPORT_SYMBOL(complete_all);
  15193. @@ -62,20 +62,20 @@
  15194. long (*action)(long), long timeout, int state)
  15195. {
  15196. if (!x->done) {
  15197. - DECLARE_WAITQUEUE(wait, current);
  15198. + DECLARE_SWAITQUEUE(wait);
  15199. - __add_wait_queue_tail_exclusive(&x->wait, &wait);
  15200. + __prepare_to_swait(&x->wait, &wait);
  15201. do {
  15202. if (signal_pending_state(state, current)) {
  15203. timeout = -ERESTARTSYS;
  15204. break;
  15205. }
  15206. __set_current_state(state);
  15207. - spin_unlock_irq(&x->wait.lock);
  15208. + raw_spin_unlock_irq(&x->wait.lock);
  15209. timeout = action(timeout);
  15210. - spin_lock_irq(&x->wait.lock);
  15211. + raw_spin_lock_irq(&x->wait.lock);
  15212. } while (!x->done && timeout);
  15213. - __remove_wait_queue(&x->wait, &wait);
  15214. + __finish_swait(&x->wait, &wait);
  15215. if (!x->done)
  15216. return timeout;
  15217. }
  15218. @@ -89,9 +89,9 @@
  15219. {
  15220. might_sleep();
  15221. - spin_lock_irq(&x->wait.lock);
  15222. + raw_spin_lock_irq(&x->wait.lock);
  15223. timeout = do_wait_for_common(x, action, timeout, state);
  15224. - spin_unlock_irq(&x->wait.lock);
  15225. + raw_spin_unlock_irq(&x->wait.lock);
  15226. return timeout;
  15227. }
  15228. @@ -277,12 +277,12 @@
  15229. if (!READ_ONCE(x->done))
  15230. return 0;
  15231. - spin_lock_irqsave(&x->wait.lock, flags);
  15232. + raw_spin_lock_irqsave(&x->wait.lock, flags);
  15233. if (!x->done)
  15234. ret = 0;
  15235. else
  15236. x->done--;
  15237. - spin_unlock_irqrestore(&x->wait.lock, flags);
  15238. + raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  15239. return ret;
  15240. }
  15241. EXPORT_SYMBOL(try_wait_for_completion);
  15242. @@ -311,7 +311,7 @@
  15243. * after it's acquired the lock.
  15244. */
  15245. smp_rmb();
  15246. - spin_unlock_wait(&x->wait.lock);
  15247. + raw_spin_unlock_wait(&x->wait.lock);
  15248. return true;
  15249. }
  15250. EXPORT_SYMBOL(completion_done);
  15251. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/core.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/core.c
  15252. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/core.c 2017-04-16 10:38:29.000000000 +0200
  15253. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/core.c 2017-04-18 17:54:26.000000000 +0200
  15254. @@ -129,7 +129,11 @@
  15255. * Number of tasks to iterate in a single balance run.
  15256. * Limited because this is done with IRQs disabled.
  15257. */
  15258. +#ifndef CONFIG_PREEMPT_RT_FULL
  15259. const_debug unsigned int sysctl_sched_nr_migrate = 32;
  15260. +#else
  15261. +const_debug unsigned int sysctl_sched_nr_migrate = 8;
  15262. +#endif
  15263. /*
  15264. * period over which we average the RT time consumption, measured
  15265. @@ -345,6 +349,7 @@
  15266. hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  15267. rq->hrtick_timer.function = hrtick;
  15268. + rq->hrtick_timer.irqsafe = 1;
  15269. }
  15270. #else /* CONFIG_SCHED_HRTICK */
  15271. static inline void hrtick_clear(struct rq *rq)
  15272. @@ -449,7 +454,7 @@
  15273. head->lastp = &node->next;
  15274. }
  15275. -void wake_up_q(struct wake_q_head *head)
  15276. +void __wake_up_q(struct wake_q_head *head, bool sleeper)
  15277. {
  15278. struct wake_q_node *node = head->first;
  15279. @@ -466,7 +471,10 @@
  15280. * wake_up_process() implies a wmb() to pair with the queueing
  15281. * in wake_q_add() so as not to miss wakeups.
  15282. */
  15283. - wake_up_process(task);
  15284. + if (sleeper)
  15285. + wake_up_lock_sleeper(task);
  15286. + else
  15287. + wake_up_process(task);
  15288. put_task_struct(task);
  15289. }
  15290. }
  15291. @@ -502,6 +510,38 @@
  15292. trace_sched_wake_idle_without_ipi(cpu);
  15293. }
  15294. +#ifdef CONFIG_PREEMPT_LAZY
  15295. +void resched_curr_lazy(struct rq *rq)
  15296. +{
  15297. + struct task_struct *curr = rq->curr;
  15298. + int cpu;
  15299. +
  15300. + if (!sched_feat(PREEMPT_LAZY)) {
  15301. + resched_curr(rq);
  15302. + return;
  15303. + }
  15304. +
  15305. + lockdep_assert_held(&rq->lock);
  15306. +
  15307. + if (test_tsk_need_resched(curr))
  15308. + return;
  15309. +
  15310. + if (test_tsk_need_resched_lazy(curr))
  15311. + return;
  15312. +
  15313. + set_tsk_need_resched_lazy(curr);
  15314. +
  15315. + cpu = cpu_of(rq);
  15316. + if (cpu == smp_processor_id())
  15317. + return;
  15318. +
  15319. + /* NEED_RESCHED_LAZY must be visible before we test polling */
  15320. + smp_mb();
  15321. + if (!tsk_is_polling(curr))
  15322. + smp_send_reschedule(cpu);
  15323. +}
  15324. +#endif
  15325. +
  15326. void resched_cpu(int cpu)
  15327. {
  15328. struct rq *rq = cpu_rq(cpu);
  15329. @@ -525,11 +565,14 @@
  15330. */
  15331. int get_nohz_timer_target(void)
  15332. {
  15333. - int i, cpu = smp_processor_id();
  15334. + int i, cpu;
  15335. struct sched_domain *sd;
  15336. + preempt_disable_rt();
  15337. + cpu = smp_processor_id();
  15338. +
  15339. if (!idle_cpu(cpu) && is_housekeeping_cpu(cpu))
  15340. - return cpu;
  15341. + goto preempt_en_rt;
  15342. rcu_read_lock();
  15343. for_each_domain(cpu, sd) {
  15344. @@ -548,6 +591,8 @@
  15345. cpu = housekeeping_any_cpu();
  15346. unlock:
  15347. rcu_read_unlock();
  15348. +preempt_en_rt:
  15349. + preempt_enable_rt();
  15350. return cpu;
  15351. }
  15352. /*
  15353. @@ -1100,6 +1145,11 @@
  15354. lockdep_assert_held(&p->pi_lock);
  15355. + if (__migrate_disabled(p)) {
  15356. + cpumask_copy(&p->cpus_allowed, new_mask);
  15357. + return;
  15358. + }
  15359. +
  15360. queued = task_on_rq_queued(p);
  15361. running = task_current(rq, p);
  15362. @@ -1122,6 +1172,84 @@
  15363. set_curr_task(rq, p);
  15364. }
  15365. +static DEFINE_PER_CPU(struct cpumask, sched_cpumasks);
  15366. +static DEFINE_MUTEX(sched_down_mutex);
  15367. +static cpumask_t sched_down_cpumask;
  15368. +
  15369. +void tell_sched_cpu_down_begin(int cpu)
  15370. +{
  15371. + mutex_lock(&sched_down_mutex);
  15372. + cpumask_set_cpu(cpu, &sched_down_cpumask);
  15373. + mutex_unlock(&sched_down_mutex);
  15374. +}
  15375. +
  15376. +void tell_sched_cpu_down_done(int cpu)
  15377. +{
  15378. + mutex_lock(&sched_down_mutex);
  15379. + cpumask_clear_cpu(cpu, &sched_down_cpumask);
  15380. + mutex_unlock(&sched_down_mutex);
  15381. +}
  15382. +
  15383. +/**
  15384. + * migrate_me - try to move the current task off this cpu
  15385. + *
  15386. + * Used by the pin_current_cpu() code to try to get tasks
  15387. + * to move off the current CPU as it is going down.
  15388. + * It will only move the task if the task isn't pinned to
  15389. + * the CPU (with migrate_disable, affinity or NO_SETAFFINITY)
  15390. + * and the task has to be in a RUNNING state. Otherwise the
  15391. + * movement of the task will wake it up (change its state
  15392. + * to running) when the task did not expect it.
  15393. + *
  15394. + * Returns 1 if it succeeded in moving the current task
  15395. + * 0 otherwise.
  15396. + */
  15397. +int migrate_me(void)
  15398. +{
  15399. + struct task_struct *p = current;
  15400. + struct migration_arg arg;
  15401. + struct cpumask *cpumask;
  15402. + struct cpumask *mask;
  15403. + unsigned int dest_cpu;
  15404. + struct rq_flags rf;
  15405. + struct rq *rq;
  15406. +
  15407. + /*
  15408. + * We can not migrate tasks bounded to a CPU or tasks not
  15409. + * running. The movement of the task will wake it up.
  15410. + */
  15411. + if (p->flags & PF_NO_SETAFFINITY || p->state)
  15412. + return 0;
  15413. +
  15414. + mutex_lock(&sched_down_mutex);
  15415. + rq = task_rq_lock(p, &rf);
  15416. +
  15417. + cpumask = this_cpu_ptr(&sched_cpumasks);
  15418. + mask = &p->cpus_allowed;
  15419. +
  15420. + cpumask_andnot(cpumask, mask, &sched_down_cpumask);
  15421. +
  15422. + if (!cpumask_weight(cpumask)) {
  15423. + /* It's only on this CPU? */
  15424. + task_rq_unlock(rq, p, &rf);
  15425. + mutex_unlock(&sched_down_mutex);
  15426. + return 0;
  15427. + }
  15428. +
  15429. + dest_cpu = cpumask_any_and(cpu_active_mask, cpumask);
  15430. +
  15431. + arg.task = p;
  15432. + arg.dest_cpu = dest_cpu;
  15433. +
  15434. + task_rq_unlock(rq, p, &rf);
  15435. +
  15436. + stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
  15437. + tlb_migrate_finish(p->mm);
  15438. + mutex_unlock(&sched_down_mutex);
  15439. +
  15440. + return 1;
  15441. +}
  15442. +
  15443. /*
  15444. * Change a given task's CPU affinity. Migrate the thread to a
  15445. * proper CPU and schedule it away if the CPU it's executing on
  15446. @@ -1179,7 +1307,7 @@
  15447. }
  15448. /* Can the task run on the task's current CPU? If so, we're done */
  15449. - if (cpumask_test_cpu(task_cpu(p), new_mask))
  15450. + if (cpumask_test_cpu(task_cpu(p), new_mask) || __migrate_disabled(p))
  15451. goto out;
  15452. dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask);
  15453. @@ -1366,6 +1494,18 @@
  15454. return ret;
  15455. }
  15456. +static bool check_task_state(struct task_struct *p, long match_state)
  15457. +{
  15458. + bool match = false;
  15459. +
  15460. + raw_spin_lock_irq(&p->pi_lock);
  15461. + if (p->state == match_state || p->saved_state == match_state)
  15462. + match = true;
  15463. + raw_spin_unlock_irq(&p->pi_lock);
  15464. +
  15465. + return match;
  15466. +}
  15467. +
  15468. /*
  15469. * wait_task_inactive - wait for a thread to unschedule.
  15470. *
  15471. @@ -1410,7 +1550,7 @@
  15472. * is actually now running somewhere else!
  15473. */
  15474. while (task_running(rq, p)) {
  15475. - if (match_state && unlikely(p->state != match_state))
  15476. + if (match_state && !check_task_state(p, match_state))
  15477. return 0;
  15478. cpu_relax();
  15479. }
  15480. @@ -1425,7 +1565,8 @@
  15481. running = task_running(rq, p);
  15482. queued = task_on_rq_queued(p);
  15483. ncsw = 0;
  15484. - if (!match_state || p->state == match_state)
  15485. + if (!match_state || p->state == match_state ||
  15486. + p->saved_state == match_state)
  15487. ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
  15488. task_rq_unlock(rq, p, &rf);
  15489. @@ -1680,10 +1821,6 @@
  15490. {
  15491. activate_task(rq, p, en_flags);
  15492. p->on_rq = TASK_ON_RQ_QUEUED;
  15493. -
  15494. - /* if a worker is waking up, notify workqueue */
  15495. - if (p->flags & PF_WQ_WORKER)
  15496. - wq_worker_waking_up(p, cpu_of(rq));
  15497. }
  15498. /*
  15499. @@ -2018,8 +2155,27 @@
  15500. */
  15501. smp_mb__before_spinlock();
  15502. raw_spin_lock_irqsave(&p->pi_lock, flags);
  15503. - if (!(p->state & state))
  15504. + if (!(p->state & state)) {
  15505. + /*
  15506. + * The task might be running due to a spinlock sleeper
  15507. + * wakeup. Check the saved state and set it to running
  15508. + * if the wakeup condition is true.
  15509. + */
  15510. + if (!(wake_flags & WF_LOCK_SLEEPER)) {
  15511. + if (p->saved_state & state) {
  15512. + p->saved_state = TASK_RUNNING;
  15513. + success = 1;
  15514. + }
  15515. + }
  15516. goto out;
  15517. + }
  15518. +
  15519. + /*
  15520. + * If this is a regular wakeup, then we can unconditionally
  15521. + * clear the saved state of a "lock sleeper".
  15522. + */
  15523. + if (!(wake_flags & WF_LOCK_SLEEPER))
  15524. + p->saved_state = TASK_RUNNING;
  15525. trace_sched_waking(p);
  15526. @@ -2102,53 +2258,6 @@
  15527. }
  15528. /**
  15529. - * try_to_wake_up_local - try to wake up a local task with rq lock held
  15530. - * @p: the thread to be awakened
  15531. - * @cookie: context's cookie for pinning
  15532. - *
  15533. - * Put @p on the run-queue if it's not already there. The caller must
  15534. - * ensure that this_rq() is locked, @p is bound to this_rq() and not
  15535. - * the current task.
  15536. - */
  15537. -static void try_to_wake_up_local(struct task_struct *p, struct pin_cookie cookie)
  15538. -{
  15539. - struct rq *rq = task_rq(p);
  15540. -
  15541. - if (WARN_ON_ONCE(rq != this_rq()) ||
  15542. - WARN_ON_ONCE(p == current))
  15543. - return;
  15544. -
  15545. - lockdep_assert_held(&rq->lock);
  15546. -
  15547. - if (!raw_spin_trylock(&p->pi_lock)) {
  15548. - /*
  15549. - * This is OK, because current is on_cpu, which avoids it being
  15550. - * picked for load-balance and preemption/IRQs are still
  15551. - * disabled avoiding further scheduler activity on it and we've
  15552. - * not yet picked a replacement task.
  15553. - */
  15554. - lockdep_unpin_lock(&rq->lock, cookie);
  15555. - raw_spin_unlock(&rq->lock);
  15556. - raw_spin_lock(&p->pi_lock);
  15557. - raw_spin_lock(&rq->lock);
  15558. - lockdep_repin_lock(&rq->lock, cookie);
  15559. - }
  15560. -
  15561. - if (!(p->state & TASK_NORMAL))
  15562. - goto out;
  15563. -
  15564. - trace_sched_waking(p);
  15565. -
  15566. - if (!task_on_rq_queued(p))
  15567. - ttwu_activate(rq, p, ENQUEUE_WAKEUP);
  15568. -
  15569. - ttwu_do_wakeup(rq, p, 0, cookie);
  15570. - ttwu_stat(p, smp_processor_id(), 0);
  15571. -out:
  15572. - raw_spin_unlock(&p->pi_lock);
  15573. -}
  15574. -
  15575. -/**
  15576. * wake_up_process - Wake up a specific process
  15577. * @p: The process to be woken up.
  15578. *
  15579. @@ -2166,6 +2275,18 @@
  15580. }
  15581. EXPORT_SYMBOL(wake_up_process);
  15582. +/**
  15583. + * wake_up_lock_sleeper - Wake up a specific process blocked on a "sleeping lock"
  15584. + * @p: The process to be woken up.
  15585. + *
  15586. + * Same as wake_up_process() above, but wake_flags=WF_LOCK_SLEEPER to indicate
  15587. + * the nature of the wakeup.
  15588. + */
  15589. +int wake_up_lock_sleeper(struct task_struct *p)
  15590. +{
  15591. + return try_to_wake_up(p, TASK_ALL, WF_LOCK_SLEEPER);
  15592. +}
  15593. +
  15594. int wake_up_state(struct task_struct *p, unsigned int state)
  15595. {
  15596. return try_to_wake_up(p, state, 0);
  15597. @@ -2442,6 +2563,9 @@
  15598. p->on_cpu = 0;
  15599. #endif
  15600. init_task_preempt_count(p);
  15601. +#ifdef CONFIG_HAVE_PREEMPT_LAZY
  15602. + task_thread_info(p)->preempt_lazy_count = 0;
  15603. +#endif
  15604. #ifdef CONFIG_SMP
  15605. plist_node_init(&p->pushable_tasks, MAX_PRIO);
  15606. RB_CLEAR_NODE(&p->pushable_dl_tasks);
  15607. @@ -2770,21 +2894,16 @@
  15608. finish_arch_post_lock_switch();
  15609. fire_sched_in_preempt_notifiers(current);
  15610. + /*
  15611. + * We use mmdrop_delayed() here so we don't have to do the
  15612. + * full __mmdrop() when we are the last user.
  15613. + */
  15614. if (mm)
  15615. - mmdrop(mm);
  15616. + mmdrop_delayed(mm);
  15617. if (unlikely(prev_state == TASK_DEAD)) {
  15618. if (prev->sched_class->task_dead)
  15619. prev->sched_class->task_dead(prev);
  15620. - /*
  15621. - * Remove function-return probe instances associated with this
  15622. - * task and put them back on the free list.
  15623. - */
  15624. - kprobe_flush_task(prev);
  15625. -
  15626. - /* Task is done with its stack. */
  15627. - put_task_stack(prev);
  15628. -
  15629. put_task_struct(prev);
  15630. }
  15631. @@ -3252,6 +3371,77 @@
  15632. schedstat_inc(this_rq()->sched_count);
  15633. }
  15634. +#if defined(CONFIG_PREEMPT_RT_FULL) && defined(CONFIG_SMP)
  15635. +
  15636. +void migrate_disable(void)
  15637. +{
  15638. + struct task_struct *p = current;
  15639. +
  15640. + if (in_atomic() || irqs_disabled()) {
  15641. +#ifdef CONFIG_SCHED_DEBUG
  15642. + p->migrate_disable_atomic++;
  15643. +#endif
  15644. + return;
  15645. + }
  15646. +
  15647. +#ifdef CONFIG_SCHED_DEBUG
  15648. + if (unlikely(p->migrate_disable_atomic)) {
  15649. + tracing_off();
  15650. + WARN_ON_ONCE(1);
  15651. + }
  15652. +#endif
  15653. +
  15654. + if (p->migrate_disable) {
  15655. + p->migrate_disable++;
  15656. + return;
  15657. + }
  15658. +
  15659. + preempt_disable();
  15660. + preempt_lazy_disable();
  15661. + pin_current_cpu();
  15662. + p->migrate_disable = 1;
  15663. + preempt_enable();
  15664. +}
  15665. +EXPORT_SYMBOL(migrate_disable);
  15666. +
  15667. +void migrate_enable(void)
  15668. +{
  15669. + struct task_struct *p = current;
  15670. +
  15671. + if (in_atomic() || irqs_disabled()) {
  15672. +#ifdef CONFIG_SCHED_DEBUG
  15673. + p->migrate_disable_atomic--;
  15674. +#endif
  15675. + return;
  15676. + }
  15677. +
  15678. +#ifdef CONFIG_SCHED_DEBUG
  15679. + if (unlikely(p->migrate_disable_atomic)) {
  15680. + tracing_off();
  15681. + WARN_ON_ONCE(1);
  15682. + }
  15683. +#endif
  15684. + WARN_ON_ONCE(p->migrate_disable <= 0);
  15685. +
  15686. + if (p->migrate_disable > 1) {
  15687. + p->migrate_disable--;
  15688. + return;
  15689. + }
  15690. +
  15691. + preempt_disable();
  15692. + /*
  15693. + * Clearing migrate_disable causes tsk_cpus_allowed to
  15694. + * show the tasks original cpu affinity.
  15695. + */
  15696. + p->migrate_disable = 0;
  15697. +
  15698. + unpin_current_cpu();
  15699. + preempt_enable();
  15700. + preempt_lazy_enable();
  15701. +}
  15702. +EXPORT_SYMBOL(migrate_enable);
  15703. +#endif
  15704. +
  15705. /*
  15706. * Pick up the highest-prio task:
  15707. */
  15708. @@ -3368,19 +3558,6 @@
  15709. } else {
  15710. deactivate_task(rq, prev, DEQUEUE_SLEEP);
  15711. prev->on_rq = 0;
  15712. -
  15713. - /*
  15714. - * If a worker went to sleep, notify and ask workqueue
  15715. - * whether it wants to wake up a task to maintain
  15716. - * concurrency.
  15717. - */
  15718. - if (prev->flags & PF_WQ_WORKER) {
  15719. - struct task_struct *to_wakeup;
  15720. -
  15721. - to_wakeup = wq_worker_sleeping(prev);
  15722. - if (to_wakeup)
  15723. - try_to_wake_up_local(to_wakeup, cookie);
  15724. - }
  15725. }
  15726. switch_count = &prev->nvcsw;
  15727. }
  15728. @@ -3390,6 +3567,7 @@
  15729. next = pick_next_task(rq, prev, cookie);
  15730. clear_tsk_need_resched(prev);
  15731. + clear_tsk_need_resched_lazy(prev);
  15732. clear_preempt_need_resched();
  15733. rq->clock_skip_update = 0;
  15734. @@ -3437,9 +3615,20 @@
  15735. static inline void sched_submit_work(struct task_struct *tsk)
  15736. {
  15737. - if (!tsk->state || tsk_is_pi_blocked(tsk))
  15738. + if (!tsk->state)
  15739. return;
  15740. /*
  15741. + * If a worker went to sleep, notify and ask workqueue whether
  15742. + * it wants to wake up a task to maintain concurrency.
  15743. + */
  15744. + if (tsk->flags & PF_WQ_WORKER)
  15745. + wq_worker_sleeping(tsk);
  15746. +
  15747. +
  15748. + if (tsk_is_pi_blocked(tsk))
  15749. + return;
  15750. +
  15751. + /*
  15752. * If we are going to sleep and we have plugged IO queued,
  15753. * make sure to submit it to avoid deadlocks.
  15754. */
  15755. @@ -3447,6 +3636,12 @@
  15756. blk_schedule_flush_plug(tsk);
  15757. }
  15758. +static void sched_update_worker(struct task_struct *tsk)
  15759. +{
  15760. + if (tsk->flags & PF_WQ_WORKER)
  15761. + wq_worker_running(tsk);
  15762. +}
  15763. +
  15764. asmlinkage __visible void __sched schedule(void)
  15765. {
  15766. struct task_struct *tsk = current;
  15767. @@ -3457,6 +3652,7 @@
  15768. __schedule(false);
  15769. sched_preempt_enable_no_resched();
  15770. } while (need_resched());
  15771. + sched_update_worker(tsk);
  15772. }
  15773. EXPORT_SYMBOL(schedule);
  15774. @@ -3520,6 +3716,30 @@
  15775. } while (need_resched());
  15776. }
  15777. +#ifdef CONFIG_PREEMPT_LAZY
  15778. +/*
  15779. + * If TIF_NEED_RESCHED is then we allow to be scheduled away since this is
  15780. + * set by a RT task. Oterwise we try to avoid beeing scheduled out as long as
  15781. + * preempt_lazy_count counter >0.
  15782. + */
  15783. +static __always_inline int preemptible_lazy(void)
  15784. +{
  15785. + if (test_thread_flag(TIF_NEED_RESCHED))
  15786. + return 1;
  15787. + if (current_thread_info()->preempt_lazy_count)
  15788. + return 0;
  15789. + return 1;
  15790. +}
  15791. +
  15792. +#else
  15793. +
  15794. +static inline int preemptible_lazy(void)
  15795. +{
  15796. + return 1;
  15797. +}
  15798. +
  15799. +#endif
  15800. +
  15801. #ifdef CONFIG_PREEMPT
  15802. /*
  15803. * this is the entry point to schedule() from in-kernel preemption
  15804. @@ -3534,7 +3754,8 @@
  15805. */
  15806. if (likely(!preemptible()))
  15807. return;
  15808. -
  15809. + if (!preemptible_lazy())
  15810. + return;
  15811. preempt_schedule_common();
  15812. }
  15813. NOKPROBE_SYMBOL(preempt_schedule);
  15814. @@ -3561,6 +3782,9 @@
  15815. if (likely(!preemptible()))
  15816. return;
  15817. + if (!preemptible_lazy())
  15818. + return;
  15819. +
  15820. do {
  15821. /*
  15822. * Because the function tracer can trace preempt_count_sub()
  15823. @@ -3583,7 +3807,16 @@
  15824. * an infinite recursion.
  15825. */
  15826. prev_ctx = exception_enter();
  15827. + /*
  15828. + * The add/subtract must not be traced by the function
  15829. + * tracer. But we still want to account for the
  15830. + * preempt off latency tracer. Since the _notrace versions
  15831. + * of add/subtract skip the accounting for latency tracer
  15832. + * we must force it manually.
  15833. + */
  15834. + start_critical_timings();
  15835. __schedule(true);
  15836. + stop_critical_timings();
  15837. exception_exit(prev_ctx);
  15838. preempt_latency_stop(1);
  15839. @@ -4939,6 +5172,7 @@
  15840. }
  15841. EXPORT_SYMBOL(__cond_resched_lock);
  15842. +#ifndef CONFIG_PREEMPT_RT_FULL
  15843. int __sched __cond_resched_softirq(void)
  15844. {
  15845. BUG_ON(!in_softirq());
  15846. @@ -4952,6 +5186,7 @@
  15847. return 0;
  15848. }
  15849. EXPORT_SYMBOL(__cond_resched_softirq);
  15850. +#endif
  15851. /**
  15852. * yield - yield the current processor to other threads.
  15853. @@ -5315,7 +5550,9 @@
  15854. /* Set the preempt count _outside_ the spinlocks! */
  15855. init_idle_preempt_count(idle, cpu);
  15856. -
  15857. +#ifdef CONFIG_HAVE_PREEMPT_LAZY
  15858. + task_thread_info(idle)->preempt_lazy_count = 0;
  15859. +#endif
  15860. /*
  15861. * The idle tasks have their own, simple scheduling class:
  15862. */
  15863. @@ -5458,6 +5695,8 @@
  15864. #endif /* CONFIG_NUMA_BALANCING */
  15865. #ifdef CONFIG_HOTPLUG_CPU
  15866. +static DEFINE_PER_CPU(struct mm_struct *, idle_last_mm);
  15867. +
  15868. /*
  15869. * Ensures that the idle task is using init_mm right before its cpu goes
  15870. * offline.
  15871. @@ -5472,7 +5711,12 @@
  15872. switch_mm_irqs_off(mm, &init_mm, current);
  15873. finish_arch_post_lock_switch();
  15874. }
  15875. - mmdrop(mm);
  15876. + /*
  15877. + * Defer the cleanup to an alive cpu. On RT we can neither
  15878. + * call mmdrop() nor mmdrop_delayed() from here.
  15879. + */
  15880. + per_cpu(idle_last_mm, smp_processor_id()) = mm;
  15881. +
  15882. }
  15883. /*
  15884. @@ -7418,6 +7662,10 @@
  15885. update_max_interval();
  15886. nohz_balance_exit_idle(cpu);
  15887. hrtick_clear(rq);
  15888. + if (per_cpu(idle_last_mm, cpu)) {
  15889. + mmdrop_delayed(per_cpu(idle_last_mm, cpu));
  15890. + per_cpu(idle_last_mm, cpu) = NULL;
  15891. + }
  15892. return 0;
  15893. }
  15894. #endif
  15895. @@ -7698,7 +7946,7 @@
  15896. #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
  15897. static inline int preempt_count_equals(int preempt_offset)
  15898. {
  15899. - int nested = preempt_count() + rcu_preempt_depth();
  15900. + int nested = preempt_count() + sched_rcu_preempt_depth();
  15901. return (nested == preempt_offset);
  15902. }
  15903. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/deadline.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/deadline.c
  15904. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/deadline.c 2017-04-16 10:38:29.000000000 +0200
  15905. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/deadline.c 2017-04-18 17:54:26.000000000 +0200
  15906. @@ -687,6 +687,7 @@
  15907. hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  15908. timer->function = dl_task_timer;
  15909. + timer->irqsafe = 1;
  15910. }
  15911. static
  15912. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/debug.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/debug.c
  15913. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/debug.c 2017-04-16 10:38:29.000000000 +0200
  15914. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/debug.c 2017-04-18 17:54:26.000000000 +0200
  15915. @@ -558,6 +558,9 @@
  15916. P(rt_throttled);
  15917. PN(rt_time);
  15918. PN(rt_runtime);
  15919. +#ifdef CONFIG_SMP
  15920. + P(rt_nr_migratory);
  15921. +#endif
  15922. #undef PN
  15923. #undef P
  15924. @@ -953,6 +956,10 @@
  15925. #endif
  15926. P(policy);
  15927. P(prio);
  15928. +#ifdef CONFIG_PREEMPT_RT_FULL
  15929. + P(migrate_disable);
  15930. +#endif
  15931. + P(nr_cpus_allowed);
  15932. #undef PN_SCHEDSTAT
  15933. #undef PN
  15934. #undef __PN
  15935. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/fair.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/fair.c
  15936. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/fair.c 2017-04-16 10:38:29.000000000 +0200
  15937. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/fair.c 2017-04-18 17:54:26.000000000 +0200
  15938. @@ -3518,7 +3518,7 @@
  15939. ideal_runtime = sched_slice(cfs_rq, curr);
  15940. delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
  15941. if (delta_exec > ideal_runtime) {
  15942. - resched_curr(rq_of(cfs_rq));
  15943. + resched_curr_lazy(rq_of(cfs_rq));
  15944. /*
  15945. * The current task ran long enough, ensure it doesn't get
  15946. * re-elected due to buddy favours.
  15947. @@ -3542,7 +3542,7 @@
  15948. return;
  15949. if (delta > ideal_runtime)
  15950. - resched_curr(rq_of(cfs_rq));
  15951. + resched_curr_lazy(rq_of(cfs_rq));
  15952. }
  15953. static void
  15954. @@ -3684,7 +3684,7 @@
  15955. * validating it and just reschedule.
  15956. */
  15957. if (queued) {
  15958. - resched_curr(rq_of(cfs_rq));
  15959. + resched_curr_lazy(rq_of(cfs_rq));
  15960. return;
  15961. }
  15962. /*
  15963. @@ -3866,7 +3866,7 @@
  15964. * hierarchy can be throttled
  15965. */
  15966. if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
  15967. - resched_curr(rq_of(cfs_rq));
  15968. + resched_curr_lazy(rq_of(cfs_rq));
  15969. }
  15970. static __always_inline
  15971. @@ -4494,7 +4494,7 @@
  15972. if (delta < 0) {
  15973. if (rq->curr == p)
  15974. - resched_curr(rq);
  15975. + resched_curr_lazy(rq);
  15976. return;
  15977. }
  15978. hrtick_start(rq, delta);
  15979. @@ -5905,7 +5905,7 @@
  15980. return;
  15981. preempt:
  15982. - resched_curr(rq);
  15983. + resched_curr_lazy(rq);
  15984. /*
  15985. * Only set the backward buddy when the current task is still
  15986. * on the rq. This can happen when a wakeup gets interleaved
  15987. @@ -8631,7 +8631,7 @@
  15988. * 'current' within the tree based on its new key value.
  15989. */
  15990. swap(curr->vruntime, se->vruntime);
  15991. - resched_curr(rq);
  15992. + resched_curr_lazy(rq);
  15993. }
  15994. se->vruntime -= cfs_rq->min_vruntime;
  15995. @@ -8655,7 +8655,7 @@
  15996. */
  15997. if (rq->curr == p) {
  15998. if (p->prio > oldprio)
  15999. - resched_curr(rq);
  16000. + resched_curr_lazy(rq);
  16001. } else
  16002. check_preempt_curr(rq, p, 0);
  16003. }
  16004. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/features.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/features.h
  16005. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/features.h 2017-04-16 10:38:29.000000000 +0200
  16006. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/features.h 2017-04-18 17:54:26.000000000 +0200
  16007. @@ -45,11 +45,19 @@
  16008. */
  16009. SCHED_FEAT(NONTASK_CAPACITY, true)
  16010. +#ifdef CONFIG_PREEMPT_RT_FULL
  16011. +SCHED_FEAT(TTWU_QUEUE, false)
  16012. +# ifdef CONFIG_PREEMPT_LAZY
  16013. +SCHED_FEAT(PREEMPT_LAZY, true)
  16014. +# endif
  16015. +#else
  16016. +
  16017. /*
  16018. * Queue remote wakeups on the target CPU and process them
  16019. * using the scheduler IPI. Reduces rq->lock contention/bounces.
  16020. */
  16021. SCHED_FEAT(TTWU_QUEUE, true)
  16022. +#endif
  16023. #ifdef HAVE_RT_PUSH_IPI
  16024. /*
  16025. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/rt.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/rt.c
  16026. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/rt.c 2017-04-16 10:38:29.000000000 +0200
  16027. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/rt.c 2017-04-18 17:54:26.000000000 +0200
  16028. @@ -47,6 +47,7 @@
  16029. hrtimer_init(&rt_b->rt_period_timer,
  16030. CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  16031. + rt_b->rt_period_timer.irqsafe = 1;
  16032. rt_b->rt_period_timer.function = sched_rt_period_timer;
  16033. }
  16034. @@ -101,6 +102,7 @@
  16035. rt_rq->push_cpu = nr_cpu_ids;
  16036. raw_spin_lock_init(&rt_rq->push_lock);
  16037. init_irq_work(&rt_rq->push_work, push_irq_work_func);
  16038. + rt_rq->push_work.flags |= IRQ_WORK_HARD_IRQ;
  16039. #endif
  16040. #endif /* CONFIG_SMP */
  16041. /* We start is dequeued state, because no RT tasks are queued */
  16042. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/sched.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/sched.h
  16043. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/sched.h 2017-04-16 10:38:29.000000000 +0200
  16044. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/sched.h 2017-04-18 17:54:26.000000000 +0200
  16045. @@ -1163,6 +1163,7 @@
  16046. #define WF_SYNC 0x01 /* waker goes to sleep after wakeup */
  16047. #define WF_FORK 0x02 /* child wakeup after fork */
  16048. #define WF_MIGRATED 0x4 /* internal use, task got migrated */
  16049. +#define WF_LOCK_SLEEPER 0x08 /* wakeup spinlock "sleeper" */
  16050. /*
  16051. * To aid in avoiding the subversion of "niceness" due to uneven distribution
  16052. @@ -1346,6 +1347,15 @@
  16053. extern void resched_curr(struct rq *rq);
  16054. extern void resched_cpu(int cpu);
  16055. +#ifdef CONFIG_PREEMPT_LAZY
  16056. +extern void resched_curr_lazy(struct rq *rq);
  16057. +#else
  16058. +static inline void resched_curr_lazy(struct rq *rq)
  16059. +{
  16060. + resched_curr(rq);
  16061. +}
  16062. +#endif
  16063. +
  16064. extern struct rt_bandwidth def_rt_bandwidth;
  16065. extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime);
  16066. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/swait.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/swait.c
  16067. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/swait.c 2017-04-16 10:38:29.000000000 +0200
  16068. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/swait.c 2017-04-18 17:54:26.000000000 +0200
  16069. @@ -1,5 +1,6 @@
  16070. #include <linux/sched.h>
  16071. #include <linux/swait.h>
  16072. +#include <linux/suspend.h>
  16073. void __init_swait_queue_head(struct swait_queue_head *q, const char *name,
  16074. struct lock_class_key *key)
  16075. @@ -29,6 +30,25 @@
  16076. }
  16077. EXPORT_SYMBOL(swake_up_locked);
  16078. +void swake_up_all_locked(struct swait_queue_head *q)
  16079. +{
  16080. + struct swait_queue *curr;
  16081. + int wakes = 0;
  16082. +
  16083. + while (!list_empty(&q->task_list)) {
  16084. +
  16085. + curr = list_first_entry(&q->task_list, typeof(*curr),
  16086. + task_list);
  16087. + wake_up_process(curr->task);
  16088. + list_del_init(&curr->task_list);
  16089. + wakes++;
  16090. + }
  16091. + if (pm_in_action)
  16092. + return;
  16093. + WARN(wakes > 2, "complete_all() with %d waiters\n", wakes);
  16094. +}
  16095. +EXPORT_SYMBOL(swake_up_all_locked);
  16096. +
  16097. void swake_up(struct swait_queue_head *q)
  16098. {
  16099. unsigned long flags;
  16100. @@ -54,6 +74,7 @@
  16101. if (!swait_active(q))
  16102. return;
  16103. + WARN_ON(irqs_disabled());
  16104. raw_spin_lock_irq(&q->lock);
  16105. list_splice_init(&q->task_list, &tmp);
  16106. while (!list_empty(&tmp)) {
  16107. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/swork.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/swork.c
  16108. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/sched/swork.c 1970-01-01 01:00:00.000000000 +0100
  16109. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/sched/swork.c 2017-04-18 17:54:26.000000000 +0200
  16110. @@ -0,0 +1,173 @@
  16111. +/*
  16112. + * Copyright (C) 2014 BMW Car IT GmbH, Daniel Wagner daniel.wagner@bmw-carit.de
  16113. + *
  16114. + * Provides a framework for enqueuing callbacks from irq context
  16115. + * PREEMPT_RT_FULL safe. The callbacks are executed in kthread context.
  16116. + */
  16117. +
  16118. +#include <linux/swait.h>
  16119. +#include <linux/swork.h>
  16120. +#include <linux/kthread.h>
  16121. +#include <linux/slab.h>
  16122. +#include <linux/spinlock.h>
  16123. +#include <linux/export.h>
  16124. +
  16125. +#define SWORK_EVENT_PENDING (1 << 0)
  16126. +
  16127. +static DEFINE_MUTEX(worker_mutex);
  16128. +static struct sworker *glob_worker;
  16129. +
  16130. +struct sworker {
  16131. + struct list_head events;
  16132. + struct swait_queue_head wq;
  16133. +
  16134. + raw_spinlock_t lock;
  16135. +
  16136. + struct task_struct *task;
  16137. + int refs;
  16138. +};
  16139. +
  16140. +static bool swork_readable(struct sworker *worker)
  16141. +{
  16142. + bool r;
  16143. +
  16144. + if (kthread_should_stop())
  16145. + return true;
  16146. +
  16147. + raw_spin_lock_irq(&worker->lock);
  16148. + r = !list_empty(&worker->events);
  16149. + raw_spin_unlock_irq(&worker->lock);
  16150. +
  16151. + return r;
  16152. +}
  16153. +
  16154. +static int swork_kthread(void *arg)
  16155. +{
  16156. + struct sworker *worker = arg;
  16157. +
  16158. + for (;;) {
  16159. + swait_event_interruptible(worker->wq,
  16160. + swork_readable(worker));
  16161. + if (kthread_should_stop())
  16162. + break;
  16163. +
  16164. + raw_spin_lock_irq(&worker->lock);
  16165. + while (!list_empty(&worker->events)) {
  16166. + struct swork_event *sev;
  16167. +
  16168. + sev = list_first_entry(&worker->events,
  16169. + struct swork_event, item);
  16170. + list_del(&sev->item);
  16171. + raw_spin_unlock_irq(&worker->lock);
  16172. +
  16173. + WARN_ON_ONCE(!test_and_clear_bit(SWORK_EVENT_PENDING,
  16174. + &sev->flags));
  16175. + sev->func(sev);
  16176. + raw_spin_lock_irq(&worker->lock);
  16177. + }
  16178. + raw_spin_unlock_irq(&worker->lock);
  16179. + }
  16180. + return 0;
  16181. +}
  16182. +
  16183. +static struct sworker *swork_create(void)
  16184. +{
  16185. + struct sworker *worker;
  16186. +
  16187. + worker = kzalloc(sizeof(*worker), GFP_KERNEL);
  16188. + if (!worker)
  16189. + return ERR_PTR(-ENOMEM);
  16190. +
  16191. + INIT_LIST_HEAD(&worker->events);
  16192. + raw_spin_lock_init(&worker->lock);
  16193. + init_swait_queue_head(&worker->wq);
  16194. +
  16195. + worker->task = kthread_run(swork_kthread, worker, "kswork");
  16196. + if (IS_ERR(worker->task)) {
  16197. + kfree(worker);
  16198. + return ERR_PTR(-ENOMEM);
  16199. + }
  16200. +
  16201. + return worker;
  16202. +}
  16203. +
  16204. +static void swork_destroy(struct sworker *worker)
  16205. +{
  16206. + kthread_stop(worker->task);
  16207. +
  16208. + WARN_ON(!list_empty(&worker->events));
  16209. + kfree(worker);
  16210. +}
  16211. +
  16212. +/**
  16213. + * swork_queue - queue swork
  16214. + *
  16215. + * Returns %false if @work was already on a queue, %true otherwise.
  16216. + *
  16217. + * The work is queued and processed on a random CPU
  16218. + */
  16219. +bool swork_queue(struct swork_event *sev)
  16220. +{
  16221. + unsigned long flags;
  16222. +
  16223. + if (test_and_set_bit(SWORK_EVENT_PENDING, &sev->flags))
  16224. + return false;
  16225. +
  16226. + raw_spin_lock_irqsave(&glob_worker->lock, flags);
  16227. + list_add_tail(&sev->item, &glob_worker->events);
  16228. + raw_spin_unlock_irqrestore(&glob_worker->lock, flags);
  16229. +
  16230. + swake_up(&glob_worker->wq);
  16231. + return true;
  16232. +}
  16233. +EXPORT_SYMBOL_GPL(swork_queue);
  16234. +
  16235. +/**
  16236. + * swork_get - get an instance of the sworker
  16237. + *
  16238. + * Returns an negative error code if the initialization if the worker did not
  16239. + * work, %0 otherwise.
  16240. + *
  16241. + */
  16242. +int swork_get(void)
  16243. +{
  16244. + struct sworker *worker;
  16245. +
  16246. + mutex_lock(&worker_mutex);
  16247. + if (!glob_worker) {
  16248. + worker = swork_create();
  16249. + if (IS_ERR(worker)) {
  16250. + mutex_unlock(&worker_mutex);
  16251. + return -ENOMEM;
  16252. + }
  16253. +
  16254. + glob_worker = worker;
  16255. + }
  16256. +
  16257. + glob_worker->refs++;
  16258. + mutex_unlock(&worker_mutex);
  16259. +
  16260. + return 0;
  16261. +}
  16262. +EXPORT_SYMBOL_GPL(swork_get);
  16263. +
  16264. +/**
  16265. + * swork_put - puts an instance of the sworker
  16266. + *
  16267. + * Will destroy the sworker thread. This function must not be called until all
  16268. + * queued events have been completed.
  16269. + */
  16270. +void swork_put(void)
  16271. +{
  16272. + mutex_lock(&worker_mutex);
  16273. +
  16274. + glob_worker->refs--;
  16275. + if (glob_worker->refs > 0)
  16276. + goto out;
  16277. +
  16278. + swork_destroy(glob_worker);
  16279. + glob_worker = NULL;
  16280. +out:
  16281. + mutex_unlock(&worker_mutex);
  16282. +}
  16283. +EXPORT_SYMBOL_GPL(swork_put);
  16284. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/signal.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/signal.c
  16285. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/signal.c 2017-04-16 10:38:30.000000000 +0200
  16286. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/signal.c 2017-04-18 17:54:26.000000000 +0200
  16287. @@ -14,6 +14,7 @@
  16288. #include <linux/export.h>
  16289. #include <linux/init.h>
  16290. #include <linux/sched.h>
  16291. +#include <linux/sched/rt.h>
  16292. #include <linux/fs.h>
  16293. #include <linux/tty.h>
  16294. #include <linux/binfmts.h>
  16295. @@ -352,13 +353,30 @@
  16296. return false;
  16297. }
  16298. +static inline struct sigqueue *get_task_cache(struct task_struct *t)
  16299. +{
  16300. + struct sigqueue *q = t->sigqueue_cache;
  16301. +
  16302. + if (cmpxchg(&t->sigqueue_cache, q, NULL) != q)
  16303. + return NULL;
  16304. + return q;
  16305. +}
  16306. +
  16307. +static inline int put_task_cache(struct task_struct *t, struct sigqueue *q)
  16308. +{
  16309. + if (cmpxchg(&t->sigqueue_cache, NULL, q) == NULL)
  16310. + return 0;
  16311. + return 1;
  16312. +}
  16313. +
  16314. /*
  16315. * allocate a new signal queue record
  16316. * - this may be called without locks if and only if t == current, otherwise an
  16317. * appropriate lock must be held to stop the target task from exiting
  16318. */
  16319. static struct sigqueue *
  16320. -__sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags, int override_rlimit)
  16321. +__sigqueue_do_alloc(int sig, struct task_struct *t, gfp_t flags,
  16322. + int override_rlimit, int fromslab)
  16323. {
  16324. struct sigqueue *q = NULL;
  16325. struct user_struct *user;
  16326. @@ -375,7 +393,10 @@
  16327. if (override_rlimit ||
  16328. atomic_read(&user->sigpending) <=
  16329. task_rlimit(t, RLIMIT_SIGPENDING)) {
  16330. - q = kmem_cache_alloc(sigqueue_cachep, flags);
  16331. + if (!fromslab)
  16332. + q = get_task_cache(t);
  16333. + if (!q)
  16334. + q = kmem_cache_alloc(sigqueue_cachep, flags);
  16335. } else {
  16336. print_dropped_signal(sig);
  16337. }
  16338. @@ -392,6 +413,13 @@
  16339. return q;
  16340. }
  16341. +static struct sigqueue *
  16342. +__sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags,
  16343. + int override_rlimit)
  16344. +{
  16345. + return __sigqueue_do_alloc(sig, t, flags, override_rlimit, 0);
  16346. +}
  16347. +
  16348. static void __sigqueue_free(struct sigqueue *q)
  16349. {
  16350. if (q->flags & SIGQUEUE_PREALLOC)
  16351. @@ -401,6 +429,21 @@
  16352. kmem_cache_free(sigqueue_cachep, q);
  16353. }
  16354. +static void sigqueue_free_current(struct sigqueue *q)
  16355. +{
  16356. + struct user_struct *up;
  16357. +
  16358. + if (q->flags & SIGQUEUE_PREALLOC)
  16359. + return;
  16360. +
  16361. + up = q->user;
  16362. + if (rt_prio(current->normal_prio) && !put_task_cache(current, q)) {
  16363. + atomic_dec(&up->sigpending);
  16364. + free_uid(up);
  16365. + } else
  16366. + __sigqueue_free(q);
  16367. +}
  16368. +
  16369. void flush_sigqueue(struct sigpending *queue)
  16370. {
  16371. struct sigqueue *q;
  16372. @@ -414,6 +457,21 @@
  16373. }
  16374. /*
  16375. + * Called from __exit_signal. Flush tsk->pending and
  16376. + * tsk->sigqueue_cache
  16377. + */
  16378. +void flush_task_sigqueue(struct task_struct *tsk)
  16379. +{
  16380. + struct sigqueue *q;
  16381. +
  16382. + flush_sigqueue(&tsk->pending);
  16383. +
  16384. + q = get_task_cache(tsk);
  16385. + if (q)
  16386. + kmem_cache_free(sigqueue_cachep, q);
  16387. +}
  16388. +
  16389. +/*
  16390. * Flush all pending signals for this kthread.
  16391. */
  16392. void flush_signals(struct task_struct *t)
  16393. @@ -525,7 +583,7 @@
  16394. still_pending:
  16395. list_del_init(&first->list);
  16396. copy_siginfo(info, &first->info);
  16397. - __sigqueue_free(first);
  16398. + sigqueue_free_current(first);
  16399. } else {
  16400. /*
  16401. * Ok, it wasn't in the queue. This must be
  16402. @@ -560,6 +618,8 @@
  16403. {
  16404. int signr;
  16405. + WARN_ON_ONCE(tsk != current);
  16406. +
  16407. /* We only dequeue private signals from ourselves, we don't let
  16408. * signalfd steal them
  16409. */
  16410. @@ -1156,8 +1216,8 @@
  16411. * We don't want to have recursive SIGSEGV's etc, for example,
  16412. * that is why we also clear SIGNAL_UNKILLABLE.
  16413. */
  16414. -int
  16415. -force_sig_info(int sig, struct siginfo *info, struct task_struct *t)
  16416. +static int
  16417. +do_force_sig_info(int sig, struct siginfo *info, struct task_struct *t)
  16418. {
  16419. unsigned long int flags;
  16420. int ret, blocked, ignored;
  16421. @@ -1182,6 +1242,39 @@
  16422. return ret;
  16423. }
  16424. +int force_sig_info(int sig, struct siginfo *info, struct task_struct *t)
  16425. +{
  16426. +/*
  16427. + * On some archs, PREEMPT_RT has to delay sending a signal from a trap
  16428. + * since it can not enable preemption, and the signal code's spin_locks
  16429. + * turn into mutexes. Instead, it must set TIF_NOTIFY_RESUME which will
  16430. + * send the signal on exit of the trap.
  16431. + */
  16432. +#ifdef ARCH_RT_DELAYS_SIGNAL_SEND
  16433. + if (in_atomic()) {
  16434. + if (WARN_ON_ONCE(t != current))
  16435. + return 0;
  16436. + if (WARN_ON_ONCE(t->forced_info.si_signo))
  16437. + return 0;
  16438. +
  16439. + if (is_si_special(info)) {
  16440. + WARN_ON_ONCE(info != SEND_SIG_PRIV);
  16441. + t->forced_info.si_signo = sig;
  16442. + t->forced_info.si_errno = 0;
  16443. + t->forced_info.si_code = SI_KERNEL;
  16444. + t->forced_info.si_pid = 0;
  16445. + t->forced_info.si_uid = 0;
  16446. + } else {
  16447. + t->forced_info = *info;
  16448. + }
  16449. +
  16450. + set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
  16451. + return 0;
  16452. + }
  16453. +#endif
  16454. + return do_force_sig_info(sig, info, t);
  16455. +}
  16456. +
  16457. /*
  16458. * Nuke all other threads in the group.
  16459. */
  16460. @@ -1216,12 +1309,12 @@
  16461. * Disable interrupts early to avoid deadlocks.
  16462. * See rcu_read_unlock() comment header for details.
  16463. */
  16464. - local_irq_save(*flags);
  16465. + local_irq_save_nort(*flags);
  16466. rcu_read_lock();
  16467. sighand = rcu_dereference(tsk->sighand);
  16468. if (unlikely(sighand == NULL)) {
  16469. rcu_read_unlock();
  16470. - local_irq_restore(*flags);
  16471. + local_irq_restore_nort(*flags);
  16472. break;
  16473. }
  16474. /*
  16475. @@ -1242,7 +1335,7 @@
  16476. }
  16477. spin_unlock(&sighand->siglock);
  16478. rcu_read_unlock();
  16479. - local_irq_restore(*flags);
  16480. + local_irq_restore_nort(*flags);
  16481. }
  16482. return sighand;
  16483. @@ -1485,7 +1578,8 @@
  16484. */
  16485. struct sigqueue *sigqueue_alloc(void)
  16486. {
  16487. - struct sigqueue *q = __sigqueue_alloc(-1, current, GFP_KERNEL, 0);
  16488. + /* Preallocated sigqueue objects always from the slabcache ! */
  16489. + struct sigqueue *q = __sigqueue_do_alloc(-1, current, GFP_KERNEL, 0, 1);
  16490. if (q)
  16491. q->flags |= SIGQUEUE_PREALLOC;
  16492. @@ -1846,15 +1940,7 @@
  16493. if (gstop_done && ptrace_reparented(current))
  16494. do_notify_parent_cldstop(current, false, why);
  16495. - /*
  16496. - * Don't want to allow preemption here, because
  16497. - * sys_ptrace() needs this task to be inactive.
  16498. - *
  16499. - * XXX: implement read_unlock_no_resched().
  16500. - */
  16501. - preempt_disable();
  16502. read_unlock(&tasklist_lock);
  16503. - preempt_enable_no_resched();
  16504. freezable_schedule();
  16505. } else {
  16506. /*
  16507. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/softirq.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/softirq.c
  16508. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/softirq.c 2017-04-16 10:38:30.000000000 +0200
  16509. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/softirq.c 2017-04-18 17:54:26.000000000 +0200
  16510. @@ -21,10 +21,12 @@
  16511. #include <linux/freezer.h>
  16512. #include <linux/kthread.h>
  16513. #include <linux/rcupdate.h>
  16514. +#include <linux/delay.h>
  16515. #include <linux/ftrace.h>
  16516. #include <linux/smp.h>
  16517. #include <linux/smpboot.h>
  16518. #include <linux/tick.h>
  16519. +#include <linux/locallock.h>
  16520. #include <linux/irq.h>
  16521. #define CREATE_TRACE_POINTS
  16522. @@ -56,12 +58,108 @@
  16523. static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  16524. DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  16525. +#ifdef CONFIG_PREEMPT_RT_FULL
  16526. +#define TIMER_SOFTIRQS ((1 << TIMER_SOFTIRQ) | (1 << HRTIMER_SOFTIRQ))
  16527. +DEFINE_PER_CPU(struct task_struct *, ktimer_softirqd);
  16528. +#endif
  16529. const char * const softirq_to_name[NR_SOFTIRQS] = {
  16530. "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
  16531. "TASKLET", "SCHED", "HRTIMER", "RCU"
  16532. };
  16533. +#ifdef CONFIG_NO_HZ_COMMON
  16534. +# ifdef CONFIG_PREEMPT_RT_FULL
  16535. +
  16536. +struct softirq_runner {
  16537. + struct task_struct *runner[NR_SOFTIRQS];
  16538. +};
  16539. +
  16540. +static DEFINE_PER_CPU(struct softirq_runner, softirq_runners);
  16541. +
  16542. +static inline void softirq_set_runner(unsigned int sirq)
  16543. +{
  16544. + struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
  16545. +
  16546. + sr->runner[sirq] = current;
  16547. +}
  16548. +
  16549. +static inline void softirq_clr_runner(unsigned int sirq)
  16550. +{
  16551. + struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
  16552. +
  16553. + sr->runner[sirq] = NULL;
  16554. +}
  16555. +
  16556. +/*
  16557. + * On preempt-rt a softirq running context might be blocked on a
  16558. + * lock. There might be no other runnable task on this CPU because the
  16559. + * lock owner runs on some other CPU. So we have to go into idle with
  16560. + * the pending bit set. Therefor we need to check this otherwise we
  16561. + * warn about false positives which confuses users and defeats the
  16562. + * whole purpose of this test.
  16563. + *
  16564. + * This code is called with interrupts disabled.
  16565. + */
  16566. +void softirq_check_pending_idle(void)
  16567. +{
  16568. + static int rate_limit;
  16569. + struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
  16570. + u32 warnpending;
  16571. + int i;
  16572. +
  16573. + if (rate_limit >= 10)
  16574. + return;
  16575. +
  16576. + warnpending = local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK;
  16577. + for (i = 0; i < NR_SOFTIRQS; i++) {
  16578. + struct task_struct *tsk = sr->runner[i];
  16579. +
  16580. + /*
  16581. + * The wakeup code in rtmutex.c wakes up the task
  16582. + * _before_ it sets pi_blocked_on to NULL under
  16583. + * tsk->pi_lock. So we need to check for both: state
  16584. + * and pi_blocked_on.
  16585. + */
  16586. + if (tsk) {
  16587. + raw_spin_lock(&tsk->pi_lock);
  16588. + if (tsk->pi_blocked_on || tsk->state == TASK_RUNNING) {
  16589. + /* Clear all bits pending in that task */
  16590. + warnpending &= ~(tsk->softirqs_raised);
  16591. + warnpending &= ~(1 << i);
  16592. + }
  16593. + raw_spin_unlock(&tsk->pi_lock);
  16594. + }
  16595. + }
  16596. +
  16597. + if (warnpending) {
  16598. + printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
  16599. + warnpending);
  16600. + rate_limit++;
  16601. + }
  16602. +}
  16603. +# else
  16604. +/*
  16605. + * On !PREEMPT_RT we just printk rate limited:
  16606. + */
  16607. +void softirq_check_pending_idle(void)
  16608. +{
  16609. + static int rate_limit;
  16610. +
  16611. + if (rate_limit < 10 &&
  16612. + (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
  16613. + printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
  16614. + local_softirq_pending());
  16615. + rate_limit++;
  16616. + }
  16617. +}
  16618. +# endif
  16619. +
  16620. +#else /* !CONFIG_NO_HZ_COMMON */
  16621. +static inline void softirq_set_runner(unsigned int sirq) { }
  16622. +static inline void softirq_clr_runner(unsigned int sirq) { }
  16623. +#endif
  16624. +
  16625. /*
  16626. * we cannot loop indefinitely here to avoid userspace starvation,
  16627. * but we also don't want to introduce a worst case 1/HZ latency
  16628. @@ -77,6 +175,38 @@
  16629. wake_up_process(tsk);
  16630. }
  16631. +#ifdef CONFIG_PREEMPT_RT_FULL
  16632. +static void wakeup_timer_softirqd(void)
  16633. +{
  16634. + /* Interrupts are disabled: no need to stop preemption */
  16635. + struct task_struct *tsk = __this_cpu_read(ktimer_softirqd);
  16636. +
  16637. + if (tsk && tsk->state != TASK_RUNNING)
  16638. + wake_up_process(tsk);
  16639. +}
  16640. +#endif
  16641. +
  16642. +static void handle_softirq(unsigned int vec_nr)
  16643. +{
  16644. + struct softirq_action *h = softirq_vec + vec_nr;
  16645. + int prev_count;
  16646. +
  16647. + prev_count = preempt_count();
  16648. +
  16649. + kstat_incr_softirqs_this_cpu(vec_nr);
  16650. +
  16651. + trace_softirq_entry(vec_nr);
  16652. + h->action(h);
  16653. + trace_softirq_exit(vec_nr);
  16654. + if (unlikely(prev_count != preempt_count())) {
  16655. + pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
  16656. + vec_nr, softirq_to_name[vec_nr], h->action,
  16657. + prev_count, preempt_count());
  16658. + preempt_count_set(prev_count);
  16659. + }
  16660. +}
  16661. +
  16662. +#ifndef CONFIG_PREEMPT_RT_FULL
  16663. /*
  16664. * If ksoftirqd is scheduled, we do not want to process pending softirqs
  16665. * right now. Let ksoftirqd handle this at its own rate, to get fairness.
  16666. @@ -88,6 +218,47 @@
  16667. return tsk && (tsk->state == TASK_RUNNING);
  16668. }
  16669. +static inline int ksoftirqd_softirq_pending(void)
  16670. +{
  16671. + return local_softirq_pending();
  16672. +}
  16673. +
  16674. +static void handle_pending_softirqs(u32 pending)
  16675. +{
  16676. + struct softirq_action *h = softirq_vec;
  16677. + int softirq_bit;
  16678. +
  16679. + local_irq_enable();
  16680. +
  16681. + h = softirq_vec;
  16682. +
  16683. + while ((softirq_bit = ffs(pending))) {
  16684. + unsigned int vec_nr;
  16685. +
  16686. + h += softirq_bit - 1;
  16687. + vec_nr = h - softirq_vec;
  16688. + handle_softirq(vec_nr);
  16689. +
  16690. + h++;
  16691. + pending >>= softirq_bit;
  16692. + }
  16693. +
  16694. + rcu_bh_qs();
  16695. + local_irq_disable();
  16696. +}
  16697. +
  16698. +static void run_ksoftirqd(unsigned int cpu)
  16699. +{
  16700. + local_irq_disable();
  16701. + if (ksoftirqd_softirq_pending()) {
  16702. + __do_softirq();
  16703. + local_irq_enable();
  16704. + cond_resched_rcu_qs();
  16705. + return;
  16706. + }
  16707. + local_irq_enable();
  16708. +}
  16709. +
  16710. /*
  16711. * preempt_count and SOFTIRQ_OFFSET usage:
  16712. * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  16713. @@ -243,10 +414,8 @@
  16714. unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
  16715. unsigned long old_flags = current->flags;
  16716. int max_restart = MAX_SOFTIRQ_RESTART;
  16717. - struct softirq_action *h;
  16718. bool in_hardirq;
  16719. __u32 pending;
  16720. - int softirq_bit;
  16721. /*
  16722. * Mask out PF_MEMALLOC s current task context is borrowed for the
  16723. @@ -265,36 +434,7 @@
  16724. /* Reset the pending bitmask before enabling irqs */
  16725. set_softirq_pending(0);
  16726. - local_irq_enable();
  16727. -
  16728. - h = softirq_vec;
  16729. -
  16730. - while ((softirq_bit = ffs(pending))) {
  16731. - unsigned int vec_nr;
  16732. - int prev_count;
  16733. -
  16734. - h += softirq_bit - 1;
  16735. -
  16736. - vec_nr = h - softirq_vec;
  16737. - prev_count = preempt_count();
  16738. -
  16739. - kstat_incr_softirqs_this_cpu(vec_nr);
  16740. -
  16741. - trace_softirq_entry(vec_nr);
  16742. - h->action(h);
  16743. - trace_softirq_exit(vec_nr);
  16744. - if (unlikely(prev_count != preempt_count())) {
  16745. - pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
  16746. - vec_nr, softirq_to_name[vec_nr], h->action,
  16747. - prev_count, preempt_count());
  16748. - preempt_count_set(prev_count);
  16749. - }
  16750. - h++;
  16751. - pending >>= softirq_bit;
  16752. - }
  16753. -
  16754. - rcu_bh_qs();
  16755. - local_irq_disable();
  16756. + handle_pending_softirqs(pending);
  16757. pending = local_softirq_pending();
  16758. if (pending) {
  16759. @@ -331,6 +471,309 @@
  16760. }
  16761. /*
  16762. + * This function must run with irqs disabled!
  16763. + */
  16764. +void raise_softirq_irqoff(unsigned int nr)
  16765. +{
  16766. + __raise_softirq_irqoff(nr);
  16767. +
  16768. + /*
  16769. + * If we're in an interrupt or softirq, we're done
  16770. + * (this also catches softirq-disabled code). We will
  16771. + * actually run the softirq once we return from
  16772. + * the irq or softirq.
  16773. + *
  16774. + * Otherwise we wake up ksoftirqd to make sure we
  16775. + * schedule the softirq soon.
  16776. + */
  16777. + if (!in_interrupt())
  16778. + wakeup_softirqd();
  16779. +}
  16780. +
  16781. +void __raise_softirq_irqoff(unsigned int nr)
  16782. +{
  16783. + trace_softirq_raise(nr);
  16784. + or_softirq_pending(1UL << nr);
  16785. +}
  16786. +
  16787. +static inline void local_bh_disable_nort(void) { local_bh_disable(); }
  16788. +static inline void _local_bh_enable_nort(void) { _local_bh_enable(); }
  16789. +static void ksoftirqd_set_sched_params(unsigned int cpu) { }
  16790. +
  16791. +#else /* !PREEMPT_RT_FULL */
  16792. +
  16793. +/*
  16794. + * On RT we serialize softirq execution with a cpu local lock per softirq
  16795. + */
  16796. +static DEFINE_PER_CPU(struct local_irq_lock [NR_SOFTIRQS], local_softirq_locks);
  16797. +
  16798. +void __init softirq_early_init(void)
  16799. +{
  16800. + int i;
  16801. +
  16802. + for (i = 0; i < NR_SOFTIRQS; i++)
  16803. + local_irq_lock_init(local_softirq_locks[i]);
  16804. +}
  16805. +
  16806. +static void lock_softirq(int which)
  16807. +{
  16808. + local_lock(local_softirq_locks[which]);
  16809. +}
  16810. +
  16811. +static void unlock_softirq(int which)
  16812. +{
  16813. + local_unlock(local_softirq_locks[which]);
  16814. +}
  16815. +
  16816. +static void do_single_softirq(int which)
  16817. +{
  16818. + unsigned long old_flags = current->flags;
  16819. +
  16820. + current->flags &= ~PF_MEMALLOC;
  16821. + vtime_account_irq_enter(current);
  16822. + current->flags |= PF_IN_SOFTIRQ;
  16823. + lockdep_softirq_enter();
  16824. + local_irq_enable();
  16825. + handle_softirq(which);
  16826. + local_irq_disable();
  16827. + lockdep_softirq_exit();
  16828. + current->flags &= ~PF_IN_SOFTIRQ;
  16829. + vtime_account_irq_enter(current);
  16830. + tsk_restore_flags(current, old_flags, PF_MEMALLOC);
  16831. +}
  16832. +
  16833. +/*
  16834. + * Called with interrupts disabled. Process softirqs which were raised
  16835. + * in current context (or on behalf of ksoftirqd).
  16836. + */
  16837. +static void do_current_softirqs(void)
  16838. +{
  16839. + while (current->softirqs_raised) {
  16840. + int i = __ffs(current->softirqs_raised);
  16841. + unsigned int pending, mask = (1U << i);
  16842. +
  16843. + current->softirqs_raised &= ~mask;
  16844. + local_irq_enable();
  16845. +
  16846. + /*
  16847. + * If the lock is contended, we boost the owner to
  16848. + * process the softirq or leave the critical section
  16849. + * now.
  16850. + */
  16851. + lock_softirq(i);
  16852. + local_irq_disable();
  16853. + softirq_set_runner(i);
  16854. + /*
  16855. + * Check with the local_softirq_pending() bits,
  16856. + * whether we need to process this still or if someone
  16857. + * else took care of it.
  16858. + */
  16859. + pending = local_softirq_pending();
  16860. + if (pending & mask) {
  16861. + set_softirq_pending(pending & ~mask);
  16862. + do_single_softirq(i);
  16863. + }
  16864. + softirq_clr_runner(i);
  16865. + WARN_ON(current->softirq_nestcnt != 1);
  16866. + local_irq_enable();
  16867. + unlock_softirq(i);
  16868. + local_irq_disable();
  16869. + }
  16870. +}
  16871. +
  16872. +void __local_bh_disable(void)
  16873. +{
  16874. + if (++current->softirq_nestcnt == 1)
  16875. + migrate_disable();
  16876. +}
  16877. +EXPORT_SYMBOL(__local_bh_disable);
  16878. +
  16879. +void __local_bh_enable(void)
  16880. +{
  16881. + if (WARN_ON(current->softirq_nestcnt == 0))
  16882. + return;
  16883. +
  16884. + local_irq_disable();
  16885. + if (current->softirq_nestcnt == 1 && current->softirqs_raised)
  16886. + do_current_softirqs();
  16887. + local_irq_enable();
  16888. +
  16889. + if (--current->softirq_nestcnt == 0)
  16890. + migrate_enable();
  16891. +}
  16892. +EXPORT_SYMBOL(__local_bh_enable);
  16893. +
  16894. +void _local_bh_enable(void)
  16895. +{
  16896. + if (WARN_ON(current->softirq_nestcnt == 0))
  16897. + return;
  16898. + if (--current->softirq_nestcnt == 0)
  16899. + migrate_enable();
  16900. +}
  16901. +EXPORT_SYMBOL(_local_bh_enable);
  16902. +
  16903. +int in_serving_softirq(void)
  16904. +{
  16905. + return current->flags & PF_IN_SOFTIRQ;
  16906. +}
  16907. +EXPORT_SYMBOL(in_serving_softirq);
  16908. +
  16909. +/* Called with preemption disabled */
  16910. +static void run_ksoftirqd(unsigned int cpu)
  16911. +{
  16912. + local_irq_disable();
  16913. + current->softirq_nestcnt++;
  16914. +
  16915. + do_current_softirqs();
  16916. + current->softirq_nestcnt--;
  16917. + local_irq_enable();
  16918. + cond_resched_rcu_qs();
  16919. +}
  16920. +
  16921. +/*
  16922. + * Called from netif_rx_ni(). Preemption enabled, but migration
  16923. + * disabled. So the cpu can't go away under us.
  16924. + */
  16925. +void thread_do_softirq(void)
  16926. +{
  16927. + if (!in_serving_softirq() && current->softirqs_raised) {
  16928. + current->softirq_nestcnt++;
  16929. + do_current_softirqs();
  16930. + current->softirq_nestcnt--;
  16931. + }
  16932. +}
  16933. +
  16934. +static void do_raise_softirq_irqoff(unsigned int nr)
  16935. +{
  16936. + unsigned int mask;
  16937. +
  16938. + mask = 1UL << nr;
  16939. +
  16940. + trace_softirq_raise(nr);
  16941. + or_softirq_pending(mask);
  16942. +
  16943. + /*
  16944. + * If we are not in a hard interrupt and inside a bh disabled
  16945. + * region, we simply raise the flag on current. local_bh_enable()
  16946. + * will make sure that the softirq is executed. Otherwise we
  16947. + * delegate it to ksoftirqd.
  16948. + */
  16949. + if (!in_irq() && current->softirq_nestcnt)
  16950. + current->softirqs_raised |= mask;
  16951. + else if (!__this_cpu_read(ksoftirqd) || !__this_cpu_read(ktimer_softirqd))
  16952. + return;
  16953. +
  16954. + if (mask & TIMER_SOFTIRQS)
  16955. + __this_cpu_read(ktimer_softirqd)->softirqs_raised |= mask;
  16956. + else
  16957. + __this_cpu_read(ksoftirqd)->softirqs_raised |= mask;
  16958. +}
  16959. +
  16960. +static void wakeup_proper_softirq(unsigned int nr)
  16961. +{
  16962. + if ((1UL << nr) & TIMER_SOFTIRQS)
  16963. + wakeup_timer_softirqd();
  16964. + else
  16965. + wakeup_softirqd();
  16966. +}
  16967. +
  16968. +void __raise_softirq_irqoff(unsigned int nr)
  16969. +{
  16970. + do_raise_softirq_irqoff(nr);
  16971. + if (!in_irq() && !current->softirq_nestcnt)
  16972. + wakeup_proper_softirq(nr);
  16973. +}
  16974. +
  16975. +/*
  16976. + * Same as __raise_softirq_irqoff() but will process them in ksoftirqd
  16977. + */
  16978. +void __raise_softirq_irqoff_ksoft(unsigned int nr)
  16979. +{
  16980. + unsigned int mask;
  16981. +
  16982. + if (WARN_ON_ONCE(!__this_cpu_read(ksoftirqd) ||
  16983. + !__this_cpu_read(ktimer_softirqd)))
  16984. + return;
  16985. + mask = 1UL << nr;
  16986. +
  16987. + trace_softirq_raise(nr);
  16988. + or_softirq_pending(mask);
  16989. + if (mask & TIMER_SOFTIRQS)
  16990. + __this_cpu_read(ktimer_softirqd)->softirqs_raised |= mask;
  16991. + else
  16992. + __this_cpu_read(ksoftirqd)->softirqs_raised |= mask;
  16993. + wakeup_proper_softirq(nr);
  16994. +}
  16995. +
  16996. +/*
  16997. + * This function must run with irqs disabled!
  16998. + */
  16999. +void raise_softirq_irqoff(unsigned int nr)
  17000. +{
  17001. + do_raise_softirq_irqoff(nr);
  17002. +
  17003. + /*
  17004. + * If we're in an hard interrupt we let irq return code deal
  17005. + * with the wakeup of ksoftirqd.
  17006. + */
  17007. + if (in_irq())
  17008. + return;
  17009. + /*
  17010. + * If we are in thread context but outside of a bh disabled
  17011. + * region, we need to wake ksoftirqd as well.
  17012. + *
  17013. + * CHECKME: Some of the places which do that could be wrapped
  17014. + * into local_bh_disable/enable pairs. Though it's unclear
  17015. + * whether this is worth the effort. To find those places just
  17016. + * raise a WARN() if the condition is met.
  17017. + */
  17018. + if (!current->softirq_nestcnt)
  17019. + wakeup_proper_softirq(nr);
  17020. +}
  17021. +
  17022. +static inline int ksoftirqd_softirq_pending(void)
  17023. +{
  17024. + return current->softirqs_raised;
  17025. +}
  17026. +
  17027. +static inline void local_bh_disable_nort(void) { }
  17028. +static inline void _local_bh_enable_nort(void) { }
  17029. +
  17030. +static inline void ksoftirqd_set_sched_params(unsigned int cpu)
  17031. +{
  17032. + /* Take over all but timer pending softirqs when starting */
  17033. + local_irq_disable();
  17034. + current->softirqs_raised = local_softirq_pending() & ~TIMER_SOFTIRQS;
  17035. + local_irq_enable();
  17036. +}
  17037. +
  17038. +static inline void ktimer_softirqd_set_sched_params(unsigned int cpu)
  17039. +{
  17040. + struct sched_param param = { .sched_priority = 1 };
  17041. +
  17042. + sched_setscheduler(current, SCHED_FIFO, &param);
  17043. +
  17044. + /* Take over timer pending softirqs when starting */
  17045. + local_irq_disable();
  17046. + current->softirqs_raised = local_softirq_pending() & TIMER_SOFTIRQS;
  17047. + local_irq_enable();
  17048. +}
  17049. +
  17050. +static inline void ktimer_softirqd_clr_sched_params(unsigned int cpu,
  17051. + bool online)
  17052. +{
  17053. + struct sched_param param = { .sched_priority = 0 };
  17054. +
  17055. + sched_setscheduler(current, SCHED_NORMAL, &param);
  17056. +}
  17057. +
  17058. +static int ktimer_softirqd_should_run(unsigned int cpu)
  17059. +{
  17060. + return current->softirqs_raised;
  17061. +}
  17062. +
  17063. +#endif /* PREEMPT_RT_FULL */
  17064. +/*
  17065. * Enter an interrupt context.
  17066. */
  17067. void irq_enter(void)
  17068. @@ -341,9 +784,9 @@
  17069. * Prevent raise_softirq from needlessly waking up ksoftirqd
  17070. * here, as softirq will be serviced on return from interrupt.
  17071. */
  17072. - local_bh_disable();
  17073. + local_bh_disable_nort();
  17074. tick_irq_enter();
  17075. - _local_bh_enable();
  17076. + _local_bh_enable_nort();
  17077. }
  17078. __irq_enter();
  17079. @@ -351,6 +794,7 @@
  17080. static inline void invoke_softirq(void)
  17081. {
  17082. +#ifndef CONFIG_PREEMPT_RT_FULL
  17083. if (ksoftirqd_running())
  17084. return;
  17085. @@ -373,6 +817,18 @@
  17086. } else {
  17087. wakeup_softirqd();
  17088. }
  17089. +#else /* PREEMPT_RT_FULL */
  17090. + unsigned long flags;
  17091. +
  17092. + local_irq_save(flags);
  17093. + if (__this_cpu_read(ksoftirqd) &&
  17094. + __this_cpu_read(ksoftirqd)->softirqs_raised)
  17095. + wakeup_softirqd();
  17096. + if (__this_cpu_read(ktimer_softirqd) &&
  17097. + __this_cpu_read(ktimer_softirqd)->softirqs_raised)
  17098. + wakeup_timer_softirqd();
  17099. + local_irq_restore(flags);
  17100. +#endif
  17101. }
  17102. static inline void tick_irq_exit(void)
  17103. @@ -409,26 +865,6 @@
  17104. trace_hardirq_exit(); /* must be last! */
  17105. }
  17106. -/*
  17107. - * This function must run with irqs disabled!
  17108. - */
  17109. -inline void raise_softirq_irqoff(unsigned int nr)
  17110. -{
  17111. - __raise_softirq_irqoff(nr);
  17112. -
  17113. - /*
  17114. - * If we're in an interrupt or softirq, we're done
  17115. - * (this also catches softirq-disabled code). We will
  17116. - * actually run the softirq once we return from
  17117. - * the irq or softirq.
  17118. - *
  17119. - * Otherwise we wake up ksoftirqd to make sure we
  17120. - * schedule the softirq soon.
  17121. - */
  17122. - if (!in_interrupt())
  17123. - wakeup_softirqd();
  17124. -}
  17125. -
  17126. void raise_softirq(unsigned int nr)
  17127. {
  17128. unsigned long flags;
  17129. @@ -438,12 +874,6 @@
  17130. local_irq_restore(flags);
  17131. }
  17132. -void __raise_softirq_irqoff(unsigned int nr)
  17133. -{
  17134. - trace_softirq_raise(nr);
  17135. - or_softirq_pending(1UL << nr);
  17136. -}
  17137. -
  17138. void open_softirq(int nr, void (*action)(struct softirq_action *))
  17139. {
  17140. softirq_vec[nr].action = action;
  17141. @@ -460,15 +890,45 @@
  17142. static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
  17143. static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
  17144. +static void inline
  17145. +__tasklet_common_schedule(struct tasklet_struct *t, struct tasklet_head *head, unsigned int nr)
  17146. +{
  17147. + if (tasklet_trylock(t)) {
  17148. +again:
  17149. + /* We may have been preempted before tasklet_trylock
  17150. + * and __tasklet_action may have already run.
  17151. + * So double check the sched bit while the takslet
  17152. + * is locked before adding it to the list.
  17153. + */
  17154. + if (test_bit(TASKLET_STATE_SCHED, &t->state)) {
  17155. + t->next = NULL;
  17156. + *head->tail = t;
  17157. + head->tail = &(t->next);
  17158. + raise_softirq_irqoff(nr);
  17159. + tasklet_unlock(t);
  17160. + } else {
  17161. + /* This is subtle. If we hit the corner case above
  17162. + * It is possible that we get preempted right here,
  17163. + * and another task has successfully called
  17164. + * tasklet_schedule(), then this function, and
  17165. + * failed on the trylock. Thus we must be sure
  17166. + * before releasing the tasklet lock, that the
  17167. + * SCHED_BIT is clear. Otherwise the tasklet
  17168. + * may get its SCHED_BIT set, but not added to the
  17169. + * list
  17170. + */
  17171. + if (!tasklet_tryunlock(t))
  17172. + goto again;
  17173. + }
  17174. + }
  17175. +}
  17176. +
  17177. void __tasklet_schedule(struct tasklet_struct *t)
  17178. {
  17179. unsigned long flags;
  17180. local_irq_save(flags);
  17181. - t->next = NULL;
  17182. - *__this_cpu_read(tasklet_vec.tail) = t;
  17183. - __this_cpu_write(tasklet_vec.tail, &(t->next));
  17184. - raise_softirq_irqoff(TASKLET_SOFTIRQ);
  17185. + __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ);
  17186. local_irq_restore(flags);
  17187. }
  17188. EXPORT_SYMBOL(__tasklet_schedule);
  17189. @@ -478,10 +938,7 @@
  17190. unsigned long flags;
  17191. local_irq_save(flags);
  17192. - t->next = NULL;
  17193. - *__this_cpu_read(tasklet_hi_vec.tail) = t;
  17194. - __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
  17195. - raise_softirq_irqoff(HI_SOFTIRQ);
  17196. + __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ);
  17197. local_irq_restore(flags);
  17198. }
  17199. EXPORT_SYMBOL(__tasklet_hi_schedule);
  17200. @@ -490,82 +947,122 @@
  17201. {
  17202. BUG_ON(!irqs_disabled());
  17203. - t->next = __this_cpu_read(tasklet_hi_vec.head);
  17204. - __this_cpu_write(tasklet_hi_vec.head, t);
  17205. - __raise_softirq_irqoff(HI_SOFTIRQ);
  17206. + __tasklet_hi_schedule(t);
  17207. }
  17208. EXPORT_SYMBOL(__tasklet_hi_schedule_first);
  17209. -static __latent_entropy void tasklet_action(struct softirq_action *a)
  17210. +void tasklet_enable(struct tasklet_struct *t)
  17211. {
  17212. - struct tasklet_struct *list;
  17213. + if (!atomic_dec_and_test(&t->count))
  17214. + return;
  17215. + if (test_and_clear_bit(TASKLET_STATE_PENDING, &t->state))
  17216. + tasklet_schedule(t);
  17217. +}
  17218. +EXPORT_SYMBOL(tasklet_enable);
  17219. - local_irq_disable();
  17220. - list = __this_cpu_read(tasklet_vec.head);
  17221. - __this_cpu_write(tasklet_vec.head, NULL);
  17222. - __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head));
  17223. - local_irq_enable();
  17224. +static void __tasklet_action(struct softirq_action *a,
  17225. + struct tasklet_struct *list)
  17226. +{
  17227. + int loops = 1000000;
  17228. while (list) {
  17229. struct tasklet_struct *t = list;
  17230. list = list->next;
  17231. - if (tasklet_trylock(t)) {
  17232. - if (!atomic_read(&t->count)) {
  17233. - if (!test_and_clear_bit(TASKLET_STATE_SCHED,
  17234. - &t->state))
  17235. - BUG();
  17236. - t->func(t->data);
  17237. - tasklet_unlock(t);
  17238. - continue;
  17239. - }
  17240. - tasklet_unlock(t);
  17241. + /*
  17242. + * Should always succeed - after a tasklist got on the
  17243. + * list (after getting the SCHED bit set from 0 to 1),
  17244. + * nothing but the tasklet softirq it got queued to can
  17245. + * lock it:
  17246. + */
  17247. + if (!tasklet_trylock(t)) {
  17248. + WARN_ON(1);
  17249. + continue;
  17250. }
  17251. - local_irq_disable();
  17252. t->next = NULL;
  17253. - *__this_cpu_read(tasklet_vec.tail) = t;
  17254. - __this_cpu_write(tasklet_vec.tail, &(t->next));
  17255. - __raise_softirq_irqoff(TASKLET_SOFTIRQ);
  17256. - local_irq_enable();
  17257. +
  17258. + /*
  17259. + * If we cannot handle the tasklet because it's disabled,
  17260. + * mark it as pending. tasklet_enable() will later
  17261. + * re-schedule the tasklet.
  17262. + */
  17263. + if (unlikely(atomic_read(&t->count))) {
  17264. +out_disabled:
  17265. + /* implicit unlock: */
  17266. + wmb();
  17267. + t->state = TASKLET_STATEF_PENDING;
  17268. + continue;
  17269. + }
  17270. +
  17271. + /*
  17272. + * After this point on the tasklet might be rescheduled
  17273. + * on another CPU, but it can only be added to another
  17274. + * CPU's tasklet list if we unlock the tasklet (which we
  17275. + * dont do yet).
  17276. + */
  17277. + if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
  17278. + WARN_ON(1);
  17279. +
  17280. +again:
  17281. + t->func(t->data);
  17282. +
  17283. + /*
  17284. + * Try to unlock the tasklet. We must use cmpxchg, because
  17285. + * another CPU might have scheduled or disabled the tasklet.
  17286. + * We only allow the STATE_RUN -> 0 transition here.
  17287. + */
  17288. + while (!tasklet_tryunlock(t)) {
  17289. + /*
  17290. + * If it got disabled meanwhile, bail out:
  17291. + */
  17292. + if (atomic_read(&t->count))
  17293. + goto out_disabled;
  17294. + /*
  17295. + * If it got scheduled meanwhile, re-execute
  17296. + * the tasklet function:
  17297. + */
  17298. + if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
  17299. + goto again;
  17300. + if (!--loops) {
  17301. + printk("hm, tasklet state: %08lx\n", t->state);
  17302. + WARN_ON(1);
  17303. + tasklet_unlock(t);
  17304. + break;
  17305. + }
  17306. + }
  17307. }
  17308. }
  17309. +static void tasklet_action(struct softirq_action *a)
  17310. +{
  17311. + struct tasklet_struct *list;
  17312. +
  17313. + local_irq_disable();
  17314. +
  17315. + list = __this_cpu_read(tasklet_vec.head);
  17316. + __this_cpu_write(tasklet_vec.head, NULL);
  17317. + __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head));
  17318. +
  17319. + local_irq_enable();
  17320. +
  17321. + __tasklet_action(a, list);
  17322. +}
  17323. +
  17324. static __latent_entropy void tasklet_hi_action(struct softirq_action *a)
  17325. {
  17326. struct tasklet_struct *list;
  17327. local_irq_disable();
  17328. +
  17329. list = __this_cpu_read(tasklet_hi_vec.head);
  17330. __this_cpu_write(tasklet_hi_vec.head, NULL);
  17331. __this_cpu_write(tasklet_hi_vec.tail, this_cpu_ptr(&tasklet_hi_vec.head));
  17332. - local_irq_enable();
  17333. - while (list) {
  17334. - struct tasklet_struct *t = list;
  17335. -
  17336. - list = list->next;
  17337. -
  17338. - if (tasklet_trylock(t)) {
  17339. - if (!atomic_read(&t->count)) {
  17340. - if (!test_and_clear_bit(TASKLET_STATE_SCHED,
  17341. - &t->state))
  17342. - BUG();
  17343. - t->func(t->data);
  17344. - tasklet_unlock(t);
  17345. - continue;
  17346. - }
  17347. - tasklet_unlock(t);
  17348. - }
  17349. + local_irq_enable();
  17350. - local_irq_disable();
  17351. - t->next = NULL;
  17352. - *__this_cpu_read(tasklet_hi_vec.tail) = t;
  17353. - __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
  17354. - __raise_softirq_irqoff(HI_SOFTIRQ);
  17355. - local_irq_enable();
  17356. - }
  17357. + __tasklet_action(a, list);
  17358. }
  17359. void tasklet_init(struct tasklet_struct *t,
  17360. @@ -586,7 +1083,7 @@
  17361. while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
  17362. do {
  17363. - yield();
  17364. + msleep(1);
  17365. } while (test_bit(TASKLET_STATE_SCHED, &t->state));
  17366. }
  17367. tasklet_unlock_wait(t);
  17368. @@ -660,25 +1157,26 @@
  17369. open_softirq(HI_SOFTIRQ, tasklet_hi_action);
  17370. }
  17371. -static int ksoftirqd_should_run(unsigned int cpu)
  17372. +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
  17373. +void tasklet_unlock_wait(struct tasklet_struct *t)
  17374. {
  17375. - return local_softirq_pending();
  17376. -}
  17377. -
  17378. -static void run_ksoftirqd(unsigned int cpu)
  17379. -{
  17380. - local_irq_disable();
  17381. - if (local_softirq_pending()) {
  17382. + while (test_bit(TASKLET_STATE_RUN, &(t)->state)) {
  17383. /*
  17384. - * We can safely run softirq on inline stack, as we are not deep
  17385. - * in the task stack here.
  17386. + * Hack for now to avoid this busy-loop:
  17387. */
  17388. - __do_softirq();
  17389. - local_irq_enable();
  17390. - cond_resched_rcu_qs();
  17391. - return;
  17392. +#ifdef CONFIG_PREEMPT_RT_FULL
  17393. + msleep(1);
  17394. +#else
  17395. + barrier();
  17396. +#endif
  17397. }
  17398. - local_irq_enable();
  17399. +}
  17400. +EXPORT_SYMBOL(tasklet_unlock_wait);
  17401. +#endif
  17402. +
  17403. +static int ksoftirqd_should_run(unsigned int cpu)
  17404. +{
  17405. + return ksoftirqd_softirq_pending();
  17406. }
  17407. #ifdef CONFIG_HOTPLUG_CPU
  17408. @@ -745,17 +1243,31 @@
  17409. static struct smp_hotplug_thread softirq_threads = {
  17410. .store = &ksoftirqd,
  17411. + .setup = ksoftirqd_set_sched_params,
  17412. .thread_should_run = ksoftirqd_should_run,
  17413. .thread_fn = run_ksoftirqd,
  17414. .thread_comm = "ksoftirqd/%u",
  17415. };
  17416. +#ifdef CONFIG_PREEMPT_RT_FULL
  17417. +static struct smp_hotplug_thread softirq_timer_threads = {
  17418. + .store = &ktimer_softirqd,
  17419. + .setup = ktimer_softirqd_set_sched_params,
  17420. + .cleanup = ktimer_softirqd_clr_sched_params,
  17421. + .thread_should_run = ktimer_softirqd_should_run,
  17422. + .thread_fn = run_ksoftirqd,
  17423. + .thread_comm = "ktimersoftd/%u",
  17424. +};
  17425. +#endif
  17426. +
  17427. static __init int spawn_ksoftirqd(void)
  17428. {
  17429. cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
  17430. takeover_tasklets);
  17431. BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
  17432. -
  17433. +#ifdef CONFIG_PREEMPT_RT_FULL
  17434. + BUG_ON(smpboot_register_percpu_thread(&softirq_timer_threads));
  17435. +#endif
  17436. return 0;
  17437. }
  17438. early_initcall(spawn_ksoftirqd);
  17439. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/stop_machine.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/stop_machine.c
  17440. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/stop_machine.c 2017-04-16 10:38:30.000000000 +0200
  17441. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/stop_machine.c 2017-04-18 17:54:26.000000000 +0200
  17442. @@ -36,7 +36,7 @@
  17443. struct cpu_stopper {
  17444. struct task_struct *thread;
  17445. - spinlock_t lock;
  17446. + raw_spinlock_t lock;
  17447. bool enabled; /* is this stopper enabled? */
  17448. struct list_head works; /* list of pending works */
  17449. @@ -78,14 +78,14 @@
  17450. unsigned long flags;
  17451. bool enabled;
  17452. - spin_lock_irqsave(&stopper->lock, flags);
  17453. + raw_spin_lock_irqsave(&stopper->lock, flags);
  17454. enabled = stopper->enabled;
  17455. if (enabled)
  17456. __cpu_stop_queue_work(stopper, work);
  17457. else if (work->done)
  17458. cpu_stop_signal_done(work->done);
  17459. - spin_unlock_irqrestore(&stopper->lock, flags);
  17460. + raw_spin_unlock_irqrestore(&stopper->lock, flags);
  17461. return enabled;
  17462. }
  17463. @@ -231,8 +231,8 @@
  17464. struct cpu_stopper *stopper2 = per_cpu_ptr(&cpu_stopper, cpu2);
  17465. int err;
  17466. retry:
  17467. - spin_lock_irq(&stopper1->lock);
  17468. - spin_lock_nested(&stopper2->lock, SINGLE_DEPTH_NESTING);
  17469. + raw_spin_lock_irq(&stopper1->lock);
  17470. + raw_spin_lock_nested(&stopper2->lock, SINGLE_DEPTH_NESTING);
  17471. err = -ENOENT;
  17472. if (!stopper1->enabled || !stopper2->enabled)
  17473. @@ -255,8 +255,8 @@
  17474. __cpu_stop_queue_work(stopper1, work1);
  17475. __cpu_stop_queue_work(stopper2, work2);
  17476. unlock:
  17477. - spin_unlock(&stopper2->lock);
  17478. - spin_unlock_irq(&stopper1->lock);
  17479. + raw_spin_unlock(&stopper2->lock);
  17480. + raw_spin_unlock_irq(&stopper1->lock);
  17481. if (unlikely(err == -EDEADLK)) {
  17482. while (stop_cpus_in_progress)
  17483. @@ -448,9 +448,9 @@
  17484. unsigned long flags;
  17485. int run;
  17486. - spin_lock_irqsave(&stopper->lock, flags);
  17487. + raw_spin_lock_irqsave(&stopper->lock, flags);
  17488. run = !list_empty(&stopper->works);
  17489. - spin_unlock_irqrestore(&stopper->lock, flags);
  17490. + raw_spin_unlock_irqrestore(&stopper->lock, flags);
  17491. return run;
  17492. }
  17493. @@ -461,13 +461,13 @@
  17494. repeat:
  17495. work = NULL;
  17496. - spin_lock_irq(&stopper->lock);
  17497. + raw_spin_lock_irq(&stopper->lock);
  17498. if (!list_empty(&stopper->works)) {
  17499. work = list_first_entry(&stopper->works,
  17500. struct cpu_stop_work, list);
  17501. list_del_init(&work->list);
  17502. }
  17503. - spin_unlock_irq(&stopper->lock);
  17504. + raw_spin_unlock_irq(&stopper->lock);
  17505. if (work) {
  17506. cpu_stop_fn_t fn = work->fn;
  17507. @@ -475,6 +475,8 @@
  17508. struct cpu_stop_done *done = work->done;
  17509. int ret;
  17510. + /* XXX */
  17511. +
  17512. /* cpu stop callbacks must not sleep, make in_atomic() == T */
  17513. preempt_count_inc();
  17514. ret = fn(arg);
  17515. @@ -541,7 +543,7 @@
  17516. for_each_possible_cpu(cpu) {
  17517. struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
  17518. - spin_lock_init(&stopper->lock);
  17519. + raw_spin_lock_init(&stopper->lock);
  17520. INIT_LIST_HEAD(&stopper->works);
  17521. }
  17522. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/hrtimer.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/hrtimer.c
  17523. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/hrtimer.c 2017-04-16 10:38:30.000000000 +0200
  17524. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/hrtimer.c 2017-04-18 17:54:26.000000000 +0200
  17525. @@ -53,6 +53,7 @@
  17526. #include <asm/uaccess.h>
  17527. #include <trace/events/timer.h>
  17528. +#include <trace/events/hist.h>
  17529. #include "tick-internal.h"
  17530. @@ -695,6 +696,29 @@
  17531. retrigger_next_event(NULL);
  17532. }
  17533. +#ifdef CONFIG_PREEMPT_RT_FULL
  17534. +
  17535. +static struct swork_event clock_set_delay_work;
  17536. +
  17537. +static void run_clock_set_delay(struct swork_event *event)
  17538. +{
  17539. + clock_was_set();
  17540. +}
  17541. +
  17542. +void clock_was_set_delayed(void)
  17543. +{
  17544. + swork_queue(&clock_set_delay_work);
  17545. +}
  17546. +
  17547. +static __init int create_clock_set_delay_thread(void)
  17548. +{
  17549. + WARN_ON(swork_get());
  17550. + INIT_SWORK(&clock_set_delay_work, run_clock_set_delay);
  17551. + return 0;
  17552. +}
  17553. +early_initcall(create_clock_set_delay_thread);
  17554. +#else /* PREEMPT_RT_FULL */
  17555. +
  17556. static void clock_was_set_work(struct work_struct *work)
  17557. {
  17558. clock_was_set();
  17559. @@ -710,6 +734,7 @@
  17560. {
  17561. schedule_work(&hrtimer_work);
  17562. }
  17563. +#endif
  17564. #else
  17565. @@ -719,11 +744,8 @@
  17566. static inline void hrtimer_switch_to_hres(void) { }
  17567. static inline void
  17568. hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { }
  17569. -static inline int hrtimer_reprogram(struct hrtimer *timer,
  17570. - struct hrtimer_clock_base *base)
  17571. -{
  17572. - return 0;
  17573. -}
  17574. +static inline void hrtimer_reprogram(struct hrtimer *timer,
  17575. + struct hrtimer_clock_base *base) { }
  17576. static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { }
  17577. static inline void retrigger_next_event(void *arg) { }
  17578. @@ -855,6 +877,32 @@
  17579. }
  17580. EXPORT_SYMBOL_GPL(hrtimer_forward);
  17581. +#ifdef CONFIG_PREEMPT_RT_BASE
  17582. +# define wake_up_timer_waiters(b) wake_up(&(b)->wait)
  17583. +
  17584. +/**
  17585. + * hrtimer_wait_for_timer - Wait for a running timer
  17586. + *
  17587. + * @timer: timer to wait for
  17588. + *
  17589. + * The function waits in case the timers callback function is
  17590. + * currently executed on the waitqueue of the timer base. The
  17591. + * waitqueue is woken up after the timer callback function has
  17592. + * finished execution.
  17593. + */
  17594. +void hrtimer_wait_for_timer(const struct hrtimer *timer)
  17595. +{
  17596. + struct hrtimer_clock_base *base = timer->base;
  17597. +
  17598. + if (base && base->cpu_base && !timer->irqsafe)
  17599. + wait_event(base->cpu_base->wait,
  17600. + !(hrtimer_callback_running(timer)));
  17601. +}
  17602. +
  17603. +#else
  17604. +# define wake_up_timer_waiters(b) do { } while (0)
  17605. +#endif
  17606. +
  17607. /*
  17608. * enqueue_hrtimer - internal function to (re)start a timer
  17609. *
  17610. @@ -896,6 +944,11 @@
  17611. if (!(state & HRTIMER_STATE_ENQUEUED))
  17612. return;
  17613. + if (unlikely(!list_empty(&timer->cb_entry))) {
  17614. + list_del_init(&timer->cb_entry);
  17615. + return;
  17616. + }
  17617. +
  17618. if (!timerqueue_del(&base->active, &timer->node))
  17619. cpu_base->active_bases &= ~(1 << base->index);
  17620. @@ -991,7 +1044,16 @@
  17621. new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED);
  17622. timer_stats_hrtimer_set_start_info(timer);
  17623. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  17624. + {
  17625. + ktime_t now = new_base->get_time();
  17626. + if (ktime_to_ns(tim) < ktime_to_ns(now))
  17627. + timer->praecox = now;
  17628. + else
  17629. + timer->praecox = ktime_set(0, 0);
  17630. + }
  17631. +#endif
  17632. leftmost = enqueue_hrtimer(timer, new_base);
  17633. if (!leftmost)
  17634. goto unlock;
  17635. @@ -1063,7 +1125,7 @@
  17636. if (ret >= 0)
  17637. return ret;
  17638. - cpu_relax();
  17639. + hrtimer_wait_for_timer(timer);
  17640. }
  17641. }
  17642. EXPORT_SYMBOL_GPL(hrtimer_cancel);
  17643. @@ -1127,6 +1189,7 @@
  17644. base = hrtimer_clockid_to_base(clock_id);
  17645. timer->base = &cpu_base->clock_base[base];
  17646. + INIT_LIST_HEAD(&timer->cb_entry);
  17647. timerqueue_init(&timer->node);
  17648. #ifdef CONFIG_TIMER_STATS
  17649. @@ -1167,6 +1230,7 @@
  17650. seq = raw_read_seqcount_begin(&cpu_base->seq);
  17651. if (timer->state != HRTIMER_STATE_INACTIVE ||
  17652. + cpu_base->running_soft == timer ||
  17653. cpu_base->running == timer)
  17654. return true;
  17655. @@ -1265,10 +1329,112 @@
  17656. cpu_base->running = NULL;
  17657. }
  17658. +#ifdef CONFIG_PREEMPT_RT_BASE
  17659. +static void hrtimer_rt_reprogram(int restart, struct hrtimer *timer,
  17660. + struct hrtimer_clock_base *base)
  17661. +{
  17662. + int leftmost;
  17663. +
  17664. + if (restart != HRTIMER_NORESTART &&
  17665. + !(timer->state & HRTIMER_STATE_ENQUEUED)) {
  17666. +
  17667. + leftmost = enqueue_hrtimer(timer, base);
  17668. + if (!leftmost)
  17669. + return;
  17670. +#ifdef CONFIG_HIGH_RES_TIMERS
  17671. + if (!hrtimer_is_hres_active(timer)) {
  17672. + /*
  17673. + * Kick to reschedule the next tick to handle the new timer
  17674. + * on dynticks target.
  17675. + */
  17676. + if (base->cpu_base->nohz_active)
  17677. + wake_up_nohz_cpu(base->cpu_base->cpu);
  17678. + } else {
  17679. +
  17680. + hrtimer_reprogram(timer, base);
  17681. + }
  17682. +#endif
  17683. + }
  17684. +}
  17685. +
  17686. +/*
  17687. + * The changes in mainline which removed the callback modes from
  17688. + * hrtimer are not yet working with -rt. The non wakeup_process()
  17689. + * based callbacks which involve sleeping locks need to be treated
  17690. + * seperately.
  17691. + */
  17692. +static void hrtimer_rt_run_pending(void)
  17693. +{
  17694. + enum hrtimer_restart (*fn)(struct hrtimer *);
  17695. + struct hrtimer_cpu_base *cpu_base;
  17696. + struct hrtimer_clock_base *base;
  17697. + struct hrtimer *timer;
  17698. + int index, restart;
  17699. +
  17700. + local_irq_disable();
  17701. + cpu_base = &per_cpu(hrtimer_bases, smp_processor_id());
  17702. +
  17703. + raw_spin_lock(&cpu_base->lock);
  17704. +
  17705. + for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) {
  17706. + base = &cpu_base->clock_base[index];
  17707. +
  17708. + while (!list_empty(&base->expired)) {
  17709. + timer = list_first_entry(&base->expired,
  17710. + struct hrtimer, cb_entry);
  17711. +
  17712. + /*
  17713. + * Same as the above __run_hrtimer function
  17714. + * just we run with interrupts enabled.
  17715. + */
  17716. + debug_deactivate(timer);
  17717. + cpu_base->running_soft = timer;
  17718. + raw_write_seqcount_barrier(&cpu_base->seq);
  17719. +
  17720. + __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);
  17721. + timer_stats_account_hrtimer(timer);
  17722. + fn = timer->function;
  17723. +
  17724. + raw_spin_unlock_irq(&cpu_base->lock);
  17725. + restart = fn(timer);
  17726. + raw_spin_lock_irq(&cpu_base->lock);
  17727. +
  17728. + hrtimer_rt_reprogram(restart, timer, base);
  17729. + raw_write_seqcount_barrier(&cpu_base->seq);
  17730. +
  17731. + WARN_ON_ONCE(cpu_base->running_soft != timer);
  17732. + cpu_base->running_soft = NULL;
  17733. + }
  17734. + }
  17735. +
  17736. + raw_spin_unlock_irq(&cpu_base->lock);
  17737. +
  17738. + wake_up_timer_waiters(cpu_base);
  17739. +}
  17740. +
  17741. +static int hrtimer_rt_defer(struct hrtimer *timer)
  17742. +{
  17743. + if (timer->irqsafe)
  17744. + return 0;
  17745. +
  17746. + __remove_hrtimer(timer, timer->base, timer->state, 0);
  17747. + list_add_tail(&timer->cb_entry, &timer->base->expired);
  17748. + return 1;
  17749. +}
  17750. +
  17751. +#else
  17752. +
  17753. +static inline int hrtimer_rt_defer(struct hrtimer *timer) { return 0; }
  17754. +
  17755. +#endif
  17756. +
  17757. +static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer);
  17758. +
  17759. static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now)
  17760. {
  17761. struct hrtimer_clock_base *base = cpu_base->clock_base;
  17762. unsigned int active = cpu_base->active_bases;
  17763. + int raise = 0;
  17764. for (; active; base++, active >>= 1) {
  17765. struct timerqueue_node *node;
  17766. @@ -1284,6 +1450,15 @@
  17767. timer = container_of(node, struct hrtimer, node);
  17768. + trace_hrtimer_interrupt(raw_smp_processor_id(),
  17769. + ktime_to_ns(ktime_sub(ktime_to_ns(timer->praecox) ?
  17770. + timer->praecox : hrtimer_get_expires(timer),
  17771. + basenow)),
  17772. + current,
  17773. + timer->function == hrtimer_wakeup ?
  17774. + container_of(timer, struct hrtimer_sleeper,
  17775. + timer)->task : NULL);
  17776. +
  17777. /*
  17778. * The immediate goal for using the softexpires is
  17779. * minimizing wakeups, not running timers at the
  17780. @@ -1299,9 +1474,14 @@
  17781. if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer))
  17782. break;
  17783. - __run_hrtimer(cpu_base, base, timer, &basenow);
  17784. + if (!hrtimer_rt_defer(timer))
  17785. + __run_hrtimer(cpu_base, base, timer, &basenow);
  17786. + else
  17787. + raise = 1;
  17788. }
  17789. }
  17790. + if (raise)
  17791. + raise_softirq_irqoff(HRTIMER_SOFTIRQ);
  17792. }
  17793. #ifdef CONFIG_HIGH_RES_TIMERS
  17794. @@ -1464,16 +1644,18 @@
  17795. void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
  17796. {
  17797. sl->timer.function = hrtimer_wakeup;
  17798. + sl->timer.irqsafe = 1;
  17799. sl->task = task;
  17800. }
  17801. EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
  17802. -static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
  17803. +static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode,
  17804. + unsigned long state)
  17805. {
  17806. hrtimer_init_sleeper(t, current);
  17807. do {
  17808. - set_current_state(TASK_INTERRUPTIBLE);
  17809. + set_current_state(state);
  17810. hrtimer_start_expires(&t->timer, mode);
  17811. if (likely(t->task))
  17812. @@ -1515,7 +1697,8 @@
  17813. HRTIMER_MODE_ABS);
  17814. hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
  17815. - if (do_nanosleep(&t, HRTIMER_MODE_ABS))
  17816. + /* cpu_chill() does not care about restart state. */
  17817. + if (do_nanosleep(&t, HRTIMER_MODE_ABS, TASK_INTERRUPTIBLE))
  17818. goto out;
  17819. rmtp = restart->nanosleep.rmtp;
  17820. @@ -1532,8 +1715,10 @@
  17821. return ret;
  17822. }
  17823. -long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
  17824. - const enum hrtimer_mode mode, const clockid_t clockid)
  17825. +static long
  17826. +__hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
  17827. + const enum hrtimer_mode mode, const clockid_t clockid,
  17828. + unsigned long state)
  17829. {
  17830. struct restart_block *restart;
  17831. struct hrtimer_sleeper t;
  17832. @@ -1546,7 +1731,7 @@
  17833. hrtimer_init_on_stack(&t.timer, clockid, mode);
  17834. hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack);
  17835. - if (do_nanosleep(&t, mode))
  17836. + if (do_nanosleep(&t, mode, state))
  17837. goto out;
  17838. /* Absolute timers do not update the rmtp value and restart: */
  17839. @@ -1573,6 +1758,12 @@
  17840. return ret;
  17841. }
  17842. +long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
  17843. + const enum hrtimer_mode mode, const clockid_t clockid)
  17844. +{
  17845. + return __hrtimer_nanosleep(rqtp, rmtp, mode, clockid, TASK_INTERRUPTIBLE);
  17846. +}
  17847. +
  17848. SYSCALL_DEFINE2(nanosleep, struct timespec __user *, rqtp,
  17849. struct timespec __user *, rmtp)
  17850. {
  17851. @@ -1587,6 +1778,26 @@
  17852. return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC);
  17853. }
  17854. +#ifdef CONFIG_PREEMPT_RT_FULL
  17855. +/*
  17856. + * Sleep for 1 ms in hope whoever holds what we want will let it go.
  17857. + */
  17858. +void cpu_chill(void)
  17859. +{
  17860. + struct timespec tu = {
  17861. + .tv_nsec = NSEC_PER_MSEC,
  17862. + };
  17863. + unsigned int freeze_flag = current->flags & PF_NOFREEZE;
  17864. +
  17865. + current->flags |= PF_NOFREEZE;
  17866. + __hrtimer_nanosleep(&tu, NULL, HRTIMER_MODE_REL, CLOCK_MONOTONIC,
  17867. + TASK_UNINTERRUPTIBLE);
  17868. + if (!freeze_flag)
  17869. + current->flags &= ~PF_NOFREEZE;
  17870. +}
  17871. +EXPORT_SYMBOL(cpu_chill);
  17872. +#endif
  17873. +
  17874. /*
  17875. * Functions related to boot-time initialization:
  17876. */
  17877. @@ -1598,10 +1809,14 @@
  17878. for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
  17879. cpu_base->clock_base[i].cpu_base = cpu_base;
  17880. timerqueue_init_head(&cpu_base->clock_base[i].active);
  17881. + INIT_LIST_HEAD(&cpu_base->clock_base[i].expired);
  17882. }
  17883. cpu_base->cpu = cpu;
  17884. hrtimer_init_hres(cpu_base);
  17885. +#ifdef CONFIG_PREEMPT_RT_BASE
  17886. + init_waitqueue_head(&cpu_base->wait);
  17887. +#endif
  17888. return 0;
  17889. }
  17890. @@ -1671,9 +1886,26 @@
  17891. #endif /* CONFIG_HOTPLUG_CPU */
  17892. +#ifdef CONFIG_PREEMPT_RT_BASE
  17893. +
  17894. +static void run_hrtimer_softirq(struct softirq_action *h)
  17895. +{
  17896. + hrtimer_rt_run_pending();
  17897. +}
  17898. +
  17899. +static void hrtimers_open_softirq(void)
  17900. +{
  17901. + open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq);
  17902. +}
  17903. +
  17904. +#else
  17905. +static void hrtimers_open_softirq(void) { }
  17906. +#endif
  17907. +
  17908. void __init hrtimers_init(void)
  17909. {
  17910. hrtimers_prepare_cpu(smp_processor_id());
  17911. + hrtimers_open_softirq();
  17912. }
  17913. /**
  17914. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/itimer.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/itimer.c
  17915. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/itimer.c 2017-04-16 10:38:30.000000000 +0200
  17916. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/itimer.c 2017-04-18 17:54:26.000000000 +0200
  17917. @@ -213,6 +213,7 @@
  17918. /* We are sharing ->siglock with it_real_fn() */
  17919. if (hrtimer_try_to_cancel(timer) < 0) {
  17920. spin_unlock_irq(&tsk->sighand->siglock);
  17921. + hrtimer_wait_for_timer(&tsk->signal->real_timer);
  17922. goto again;
  17923. }
  17924. expires = timeval_to_ktime(value->it_value);
  17925. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/jiffies.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/jiffies.c
  17926. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/jiffies.c 2017-04-16 10:38:30.000000000 +0200
  17927. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/jiffies.c 2017-04-18 17:54:26.000000000 +0200
  17928. @@ -74,7 +74,8 @@
  17929. .max_cycles = 10,
  17930. };
  17931. -__cacheline_aligned_in_smp DEFINE_SEQLOCK(jiffies_lock);
  17932. +__cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(jiffies_lock);
  17933. +__cacheline_aligned_in_smp seqcount_t jiffies_seq;
  17934. #if (BITS_PER_LONG < 64)
  17935. u64 get_jiffies_64(void)
  17936. @@ -83,9 +84,9 @@
  17937. u64 ret;
  17938. do {
  17939. - seq = read_seqbegin(&jiffies_lock);
  17940. + seq = read_seqcount_begin(&jiffies_seq);
  17941. ret = jiffies_64;
  17942. - } while (read_seqretry(&jiffies_lock, seq));
  17943. + } while (read_seqcount_retry(&jiffies_seq, seq));
  17944. return ret;
  17945. }
  17946. EXPORT_SYMBOL(get_jiffies_64);
  17947. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/ntp.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/ntp.c
  17948. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/ntp.c 2017-04-16 10:38:30.000000000 +0200
  17949. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/ntp.c 2017-04-18 17:54:26.000000000 +0200
  17950. @@ -17,6 +17,7 @@
  17951. #include <linux/module.h>
  17952. #include <linux/rtc.h>
  17953. #include <linux/math64.h>
  17954. +#include <linux/swork.h>
  17955. #include "ntp_internal.h"
  17956. #include "timekeeping_internal.h"
  17957. @@ -568,10 +569,35 @@
  17958. &sync_cmos_work, timespec64_to_jiffies(&next));
  17959. }
  17960. +#ifdef CONFIG_PREEMPT_RT_FULL
  17961. +
  17962. +static void run_clock_set_delay(struct swork_event *event)
  17963. +{
  17964. + queue_delayed_work(system_power_efficient_wq, &sync_cmos_work, 0);
  17965. +}
  17966. +
  17967. +static struct swork_event ntp_cmos_swork;
  17968. +
  17969. +void ntp_notify_cmos_timer(void)
  17970. +{
  17971. + swork_queue(&ntp_cmos_swork);
  17972. +}
  17973. +
  17974. +static __init int create_cmos_delay_thread(void)
  17975. +{
  17976. + WARN_ON(swork_get());
  17977. + INIT_SWORK(&ntp_cmos_swork, run_clock_set_delay);
  17978. + return 0;
  17979. +}
  17980. +early_initcall(create_cmos_delay_thread);
  17981. +
  17982. +#else
  17983. +
  17984. void ntp_notify_cmos_timer(void)
  17985. {
  17986. queue_delayed_work(system_power_efficient_wq, &sync_cmos_work, 0);
  17987. }
  17988. +#endif /* CONFIG_PREEMPT_RT_FULL */
  17989. #else
  17990. void ntp_notify_cmos_timer(void) { }
  17991. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/posix-cpu-timers.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/posix-cpu-timers.c
  17992. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/posix-cpu-timers.c 2017-04-16 10:38:30.000000000 +0200
  17993. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/posix-cpu-timers.c 2017-04-18 17:54:26.000000000 +0200
  17994. @@ -3,6 +3,7 @@
  17995. */
  17996. #include <linux/sched.h>
  17997. +#include <linux/sched/rt.h>
  17998. #include <linux/posix-timers.h>
  17999. #include <linux/errno.h>
  18000. #include <linux/math64.h>
  18001. @@ -620,7 +621,7 @@
  18002. /*
  18003. * Disarm any old timer after extracting its expiry time.
  18004. */
  18005. - WARN_ON_ONCE(!irqs_disabled());
  18006. + WARN_ON_ONCE_NONRT(!irqs_disabled());
  18007. ret = 0;
  18008. old_incr = timer->it.cpu.incr;
  18009. @@ -1064,7 +1065,7 @@
  18010. /*
  18011. * Now re-arm for the new expiry time.
  18012. */
  18013. - WARN_ON_ONCE(!irqs_disabled());
  18014. + WARN_ON_ONCE_NONRT(!irqs_disabled());
  18015. arm_timer(timer);
  18016. unlock_task_sighand(p, &flags);
  18017. @@ -1153,13 +1154,13 @@
  18018. * already updated our counts. We need to check if any timers fire now.
  18019. * Interrupts are disabled.
  18020. */
  18021. -void run_posix_cpu_timers(struct task_struct *tsk)
  18022. +static void __run_posix_cpu_timers(struct task_struct *tsk)
  18023. {
  18024. LIST_HEAD(firing);
  18025. struct k_itimer *timer, *next;
  18026. unsigned long flags;
  18027. - WARN_ON_ONCE(!irqs_disabled());
  18028. + WARN_ON_ONCE_NONRT(!irqs_disabled());
  18029. /*
  18030. * The fast path checks that there are no expired thread or thread
  18031. @@ -1213,6 +1214,190 @@
  18032. }
  18033. }
  18034. +#ifdef CONFIG_PREEMPT_RT_BASE
  18035. +#include <linux/kthread.h>
  18036. +#include <linux/cpu.h>
  18037. +DEFINE_PER_CPU(struct task_struct *, posix_timer_task);
  18038. +DEFINE_PER_CPU(struct task_struct *, posix_timer_tasklist);
  18039. +
  18040. +static int posix_cpu_timers_thread(void *data)
  18041. +{
  18042. + int cpu = (long)data;
  18043. +
  18044. + BUG_ON(per_cpu(posix_timer_task,cpu) != current);
  18045. +
  18046. + while (!kthread_should_stop()) {
  18047. + struct task_struct *tsk = NULL;
  18048. + struct task_struct *next = NULL;
  18049. +
  18050. + if (cpu_is_offline(cpu))
  18051. + goto wait_to_die;
  18052. +
  18053. + /* grab task list */
  18054. + raw_local_irq_disable();
  18055. + tsk = per_cpu(posix_timer_tasklist, cpu);
  18056. + per_cpu(posix_timer_tasklist, cpu) = NULL;
  18057. + raw_local_irq_enable();
  18058. +
  18059. + /* its possible the list is empty, just return */
  18060. + if (!tsk) {
  18061. + set_current_state(TASK_INTERRUPTIBLE);
  18062. + schedule();
  18063. + __set_current_state(TASK_RUNNING);
  18064. + continue;
  18065. + }
  18066. +
  18067. + /* Process task list */
  18068. + while (1) {
  18069. + /* save next */
  18070. + next = tsk->posix_timer_list;
  18071. +
  18072. + /* run the task timers, clear its ptr and
  18073. + * unreference it
  18074. + */
  18075. + __run_posix_cpu_timers(tsk);
  18076. + tsk->posix_timer_list = NULL;
  18077. + put_task_struct(tsk);
  18078. +
  18079. + /* check if this is the last on the list */
  18080. + if (next == tsk)
  18081. + break;
  18082. + tsk = next;
  18083. + }
  18084. + }
  18085. + return 0;
  18086. +
  18087. +wait_to_die:
  18088. + /* Wait for kthread_stop */
  18089. + set_current_state(TASK_INTERRUPTIBLE);
  18090. + while (!kthread_should_stop()) {
  18091. + schedule();
  18092. + set_current_state(TASK_INTERRUPTIBLE);
  18093. + }
  18094. + __set_current_state(TASK_RUNNING);
  18095. + return 0;
  18096. +}
  18097. +
  18098. +static inline int __fastpath_timer_check(struct task_struct *tsk)
  18099. +{
  18100. + /* tsk == current, ensure it is safe to use ->signal/sighand */
  18101. + if (unlikely(tsk->exit_state))
  18102. + return 0;
  18103. +
  18104. + if (!task_cputime_zero(&tsk->cputime_expires))
  18105. + return 1;
  18106. +
  18107. + if (!task_cputime_zero(&tsk->signal->cputime_expires))
  18108. + return 1;
  18109. +
  18110. + return 0;
  18111. +}
  18112. +
  18113. +void run_posix_cpu_timers(struct task_struct *tsk)
  18114. +{
  18115. + unsigned long cpu = smp_processor_id();
  18116. + struct task_struct *tasklist;
  18117. +
  18118. + BUG_ON(!irqs_disabled());
  18119. + if(!per_cpu(posix_timer_task, cpu))
  18120. + return;
  18121. + /* get per-cpu references */
  18122. + tasklist = per_cpu(posix_timer_tasklist, cpu);
  18123. +
  18124. + /* check to see if we're already queued */
  18125. + if (!tsk->posix_timer_list && __fastpath_timer_check(tsk)) {
  18126. + get_task_struct(tsk);
  18127. + if (tasklist) {
  18128. + tsk->posix_timer_list = tasklist;
  18129. + } else {
  18130. + /*
  18131. + * The list is terminated by a self-pointing
  18132. + * task_struct
  18133. + */
  18134. + tsk->posix_timer_list = tsk;
  18135. + }
  18136. + per_cpu(posix_timer_tasklist, cpu) = tsk;
  18137. +
  18138. + wake_up_process(per_cpu(posix_timer_task, cpu));
  18139. + }
  18140. +}
  18141. +
  18142. +/*
  18143. + * posix_cpu_thread_call - callback that gets triggered when a CPU is added.
  18144. + * Here we can start up the necessary migration thread for the new CPU.
  18145. + */
  18146. +static int posix_cpu_thread_call(struct notifier_block *nfb,
  18147. + unsigned long action, void *hcpu)
  18148. +{
  18149. + int cpu = (long)hcpu;
  18150. + struct task_struct *p;
  18151. + struct sched_param param;
  18152. +
  18153. + switch (action) {
  18154. + case CPU_UP_PREPARE:
  18155. + p = kthread_create(posix_cpu_timers_thread, hcpu,
  18156. + "posixcputmr/%d",cpu);
  18157. + if (IS_ERR(p))
  18158. + return NOTIFY_BAD;
  18159. + p->flags |= PF_NOFREEZE;
  18160. + kthread_bind(p, cpu);
  18161. + /* Must be high prio to avoid getting starved */
  18162. + param.sched_priority = MAX_RT_PRIO-1;
  18163. + sched_setscheduler(p, SCHED_FIFO, &param);
  18164. + per_cpu(posix_timer_task,cpu) = p;
  18165. + break;
  18166. + case CPU_ONLINE:
  18167. + /* Strictly unneccessary, as first user will wake it. */
  18168. + wake_up_process(per_cpu(posix_timer_task,cpu));
  18169. + break;
  18170. +#ifdef CONFIG_HOTPLUG_CPU
  18171. + case CPU_UP_CANCELED:
  18172. + /* Unbind it from offline cpu so it can run. Fall thru. */
  18173. + kthread_bind(per_cpu(posix_timer_task, cpu),
  18174. + cpumask_any(cpu_online_mask));
  18175. + kthread_stop(per_cpu(posix_timer_task,cpu));
  18176. + per_cpu(posix_timer_task,cpu) = NULL;
  18177. + break;
  18178. + case CPU_DEAD:
  18179. + kthread_stop(per_cpu(posix_timer_task,cpu));
  18180. + per_cpu(posix_timer_task,cpu) = NULL;
  18181. + break;
  18182. +#endif
  18183. + }
  18184. + return NOTIFY_OK;
  18185. +}
  18186. +
  18187. +/* Register at highest priority so that task migration (migrate_all_tasks)
  18188. + * happens before everything else.
  18189. + */
  18190. +static struct notifier_block posix_cpu_thread_notifier = {
  18191. + .notifier_call = posix_cpu_thread_call,
  18192. + .priority = 10
  18193. +};
  18194. +
  18195. +static int __init posix_cpu_thread_init(void)
  18196. +{
  18197. + void *hcpu = (void *)(long)smp_processor_id();
  18198. + /* Start one for boot CPU. */
  18199. + unsigned long cpu;
  18200. +
  18201. + /* init the per-cpu posix_timer_tasklets */
  18202. + for_each_possible_cpu(cpu)
  18203. + per_cpu(posix_timer_tasklist, cpu) = NULL;
  18204. +
  18205. + posix_cpu_thread_call(&posix_cpu_thread_notifier, CPU_UP_PREPARE, hcpu);
  18206. + posix_cpu_thread_call(&posix_cpu_thread_notifier, CPU_ONLINE, hcpu);
  18207. + register_cpu_notifier(&posix_cpu_thread_notifier);
  18208. + return 0;
  18209. +}
  18210. +early_initcall(posix_cpu_thread_init);
  18211. +#else /* CONFIG_PREEMPT_RT_BASE */
  18212. +void run_posix_cpu_timers(struct task_struct *tsk)
  18213. +{
  18214. + __run_posix_cpu_timers(tsk);
  18215. +}
  18216. +#endif /* CONFIG_PREEMPT_RT_BASE */
  18217. +
  18218. /*
  18219. * Set one of the process-wide special case CPU timers or RLIMIT_CPU.
  18220. * The tsk->sighand->siglock must be held by the caller.
  18221. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/posix-timers.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/posix-timers.c
  18222. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/posix-timers.c 2017-04-16 10:38:30.000000000 +0200
  18223. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/posix-timers.c 2017-04-18 17:54:26.000000000 +0200
  18224. @@ -506,6 +506,7 @@
  18225. static struct pid *good_sigevent(sigevent_t * event)
  18226. {
  18227. struct task_struct *rtn = current->group_leader;
  18228. + int sig = event->sigev_signo;
  18229. if ((event->sigev_notify & SIGEV_THREAD_ID ) &&
  18230. (!(rtn = find_task_by_vpid(event->sigev_notify_thread_id)) ||
  18231. @@ -514,7 +515,8 @@
  18232. return NULL;
  18233. if (((event->sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_NONE) &&
  18234. - ((event->sigev_signo <= 0) || (event->sigev_signo > SIGRTMAX)))
  18235. + (sig <= 0 || sig > SIGRTMAX || sig_kernel_only(sig) ||
  18236. + sig_kernel_coredump(sig)))
  18237. return NULL;
  18238. return task_pid(rtn);
  18239. @@ -826,6 +828,20 @@
  18240. return overrun;
  18241. }
  18242. +/*
  18243. + * Protected by RCU!
  18244. + */
  18245. +static void timer_wait_for_callback(struct k_clock *kc, struct k_itimer *timr)
  18246. +{
  18247. +#ifdef CONFIG_PREEMPT_RT_FULL
  18248. + if (kc->timer_set == common_timer_set)
  18249. + hrtimer_wait_for_timer(&timr->it.real.timer);
  18250. + else
  18251. + /* FIXME: Whacky hack for posix-cpu-timers */
  18252. + schedule_timeout(1);
  18253. +#endif
  18254. +}
  18255. +
  18256. /* Set a POSIX.1b interval timer. */
  18257. /* timr->it_lock is taken. */
  18258. static int
  18259. @@ -903,6 +919,7 @@
  18260. if (!timr)
  18261. return -EINVAL;
  18262. + rcu_read_lock();
  18263. kc = clockid_to_kclock(timr->it_clock);
  18264. if (WARN_ON_ONCE(!kc || !kc->timer_set))
  18265. error = -EINVAL;
  18266. @@ -911,9 +928,12 @@
  18267. unlock_timer(timr, flag);
  18268. if (error == TIMER_RETRY) {
  18269. + timer_wait_for_callback(kc, timr);
  18270. rtn = NULL; // We already got the old time...
  18271. + rcu_read_unlock();
  18272. goto retry;
  18273. }
  18274. + rcu_read_unlock();
  18275. if (old_setting && !error &&
  18276. copy_to_user(old_setting, &old_spec, sizeof (old_spec)))
  18277. @@ -951,10 +971,15 @@
  18278. if (!timer)
  18279. return -EINVAL;
  18280. + rcu_read_lock();
  18281. if (timer_delete_hook(timer) == TIMER_RETRY) {
  18282. unlock_timer(timer, flags);
  18283. + timer_wait_for_callback(clockid_to_kclock(timer->it_clock),
  18284. + timer);
  18285. + rcu_read_unlock();
  18286. goto retry_delete;
  18287. }
  18288. + rcu_read_unlock();
  18289. spin_lock(&current->sighand->siglock);
  18290. list_del(&timer->list);
  18291. @@ -980,8 +1005,18 @@
  18292. retry_delete:
  18293. spin_lock_irqsave(&timer->it_lock, flags);
  18294. + /* On RT we can race with a deletion */
  18295. + if (!timer->it_signal) {
  18296. + unlock_timer(timer, flags);
  18297. + return;
  18298. + }
  18299. +
  18300. if (timer_delete_hook(timer) == TIMER_RETRY) {
  18301. + rcu_read_lock();
  18302. unlock_timer(timer, flags);
  18303. + timer_wait_for_callback(clockid_to_kclock(timer->it_clock),
  18304. + timer);
  18305. + rcu_read_unlock();
  18306. goto retry_delete;
  18307. }
  18308. list_del(&timer->list);
  18309. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/tick-broadcast-hrtimer.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/tick-broadcast-hrtimer.c
  18310. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/tick-broadcast-hrtimer.c 2017-04-16 10:38:30.000000000 +0200
  18311. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/tick-broadcast-hrtimer.c 2017-04-18 17:54:26.000000000 +0200
  18312. @@ -107,5 +107,6 @@
  18313. {
  18314. hrtimer_init(&bctimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
  18315. bctimer.function = bc_handler;
  18316. + bctimer.irqsafe = true;
  18317. clockevents_register_device(&ce_broadcast_hrtimer);
  18318. }
  18319. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/tick-common.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/tick-common.c
  18320. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/tick-common.c 2017-04-16 10:38:30.000000000 +0200
  18321. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/tick-common.c 2017-04-18 17:54:26.000000000 +0200
  18322. @@ -79,13 +79,15 @@
  18323. static void tick_periodic(int cpu)
  18324. {
  18325. if (tick_do_timer_cpu == cpu) {
  18326. - write_seqlock(&jiffies_lock);
  18327. + raw_spin_lock(&jiffies_lock);
  18328. + write_seqcount_begin(&jiffies_seq);
  18329. /* Keep track of the next tick event */
  18330. tick_next_period = ktime_add(tick_next_period, tick_period);
  18331. do_timer(1);
  18332. - write_sequnlock(&jiffies_lock);
  18333. + write_seqcount_end(&jiffies_seq);
  18334. + raw_spin_unlock(&jiffies_lock);
  18335. update_wall_time();
  18336. }
  18337. @@ -157,9 +159,9 @@
  18338. ktime_t next;
  18339. do {
  18340. - seq = read_seqbegin(&jiffies_lock);
  18341. + seq = read_seqcount_begin(&jiffies_seq);
  18342. next = tick_next_period;
  18343. - } while (read_seqretry(&jiffies_lock, seq));
  18344. + } while (read_seqcount_retry(&jiffies_seq, seq));
  18345. clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
  18346. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/tick-sched.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/tick-sched.c
  18347. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/tick-sched.c 2017-04-16 10:38:30.000000000 +0200
  18348. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/tick-sched.c 2017-04-18 17:54:26.000000000 +0200
  18349. @@ -62,7 +62,8 @@
  18350. return;
  18351. /* Reevaluate with jiffies_lock held */
  18352. - write_seqlock(&jiffies_lock);
  18353. + raw_spin_lock(&jiffies_lock);
  18354. + write_seqcount_begin(&jiffies_seq);
  18355. delta = ktime_sub(now, last_jiffies_update);
  18356. if (delta.tv64 >= tick_period.tv64) {
  18357. @@ -85,10 +86,12 @@
  18358. /* Keep the tick_next_period variable up to date */
  18359. tick_next_period = ktime_add(last_jiffies_update, tick_period);
  18360. } else {
  18361. - write_sequnlock(&jiffies_lock);
  18362. + write_seqcount_end(&jiffies_seq);
  18363. + raw_spin_unlock(&jiffies_lock);
  18364. return;
  18365. }
  18366. - write_sequnlock(&jiffies_lock);
  18367. + write_seqcount_end(&jiffies_seq);
  18368. + raw_spin_unlock(&jiffies_lock);
  18369. update_wall_time();
  18370. }
  18371. @@ -99,12 +102,14 @@
  18372. {
  18373. ktime_t period;
  18374. - write_seqlock(&jiffies_lock);
  18375. + raw_spin_lock(&jiffies_lock);
  18376. + write_seqcount_begin(&jiffies_seq);
  18377. /* Did we start the jiffies update yet ? */
  18378. if (last_jiffies_update.tv64 == 0)
  18379. last_jiffies_update = tick_next_period;
  18380. period = last_jiffies_update;
  18381. - write_sequnlock(&jiffies_lock);
  18382. + write_seqcount_end(&jiffies_seq);
  18383. + raw_spin_unlock(&jiffies_lock);
  18384. return period;
  18385. }
  18386. @@ -215,6 +220,7 @@
  18387. static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
  18388. .func = nohz_full_kick_func,
  18389. + .flags = IRQ_WORK_HARD_IRQ,
  18390. };
  18391. /*
  18392. @@ -673,10 +679,10 @@
  18393. /* Read jiffies and the time when jiffies were updated last */
  18394. do {
  18395. - seq = read_seqbegin(&jiffies_lock);
  18396. + seq = read_seqcount_begin(&jiffies_seq);
  18397. basemono = last_jiffies_update.tv64;
  18398. basejiff = jiffies;
  18399. - } while (read_seqretry(&jiffies_lock, seq));
  18400. + } while (read_seqcount_retry(&jiffies_seq, seq));
  18401. ts->last_jiffies = basejiff;
  18402. if (rcu_needs_cpu(basemono, &next_rcu) ||
  18403. @@ -877,14 +883,7 @@
  18404. return false;
  18405. if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
  18406. - static int ratelimit;
  18407. -
  18408. - if (ratelimit < 10 &&
  18409. - (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
  18410. - pr_warn("NOHZ: local_softirq_pending %02x\n",
  18411. - (unsigned int) local_softirq_pending());
  18412. - ratelimit++;
  18413. - }
  18414. + softirq_check_pending_idle();
  18415. return false;
  18416. }
  18417. @@ -1193,6 +1192,7 @@
  18418. * Emulate tick processing via per-CPU hrtimers:
  18419. */
  18420. hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
  18421. + ts->sched_timer.irqsafe = 1;
  18422. ts->sched_timer.function = tick_sched_timer;
  18423. /* Get the next period (per-CPU) */
  18424. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/timekeeping.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/timekeeping.c
  18425. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/timekeeping.c 2017-04-16 10:38:30.000000000 +0200
  18426. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/timekeeping.c 2017-04-18 17:54:26.000000000 +0200
  18427. @@ -2328,8 +2328,10 @@
  18428. */
  18429. void xtime_update(unsigned long ticks)
  18430. {
  18431. - write_seqlock(&jiffies_lock);
  18432. + raw_spin_lock(&jiffies_lock);
  18433. + write_seqcount_begin(&jiffies_seq);
  18434. do_timer(ticks);
  18435. - write_sequnlock(&jiffies_lock);
  18436. + write_seqcount_end(&jiffies_seq);
  18437. + raw_spin_unlock(&jiffies_lock);
  18438. update_wall_time();
  18439. }
  18440. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/timekeeping.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/timekeeping.h
  18441. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/timekeeping.h 2017-04-16 10:38:30.000000000 +0200
  18442. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/timekeeping.h 2017-04-18 17:54:26.000000000 +0200
  18443. @@ -19,7 +19,8 @@
  18444. extern void do_timer(unsigned long ticks);
  18445. extern void update_wall_time(void);
  18446. -extern seqlock_t jiffies_lock;
  18447. +extern raw_spinlock_t jiffies_lock;
  18448. +extern seqcount_t jiffies_seq;
  18449. #define CS_NAME_LEN 32
  18450. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/timer.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/timer.c
  18451. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/time/timer.c 2017-04-16 10:38:30.000000000 +0200
  18452. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/time/timer.c 2017-04-18 17:54:26.000000000 +0200
  18453. @@ -193,8 +193,11 @@
  18454. #endif
  18455. struct timer_base {
  18456. - spinlock_t lock;
  18457. + raw_spinlock_t lock;
  18458. struct timer_list *running_timer;
  18459. +#ifdef CONFIG_PREEMPT_RT_FULL
  18460. + struct swait_queue_head wait_for_running_timer;
  18461. +#endif
  18462. unsigned long clk;
  18463. unsigned long next_expiry;
  18464. unsigned int cpu;
  18465. @@ -203,6 +206,8 @@
  18466. bool is_idle;
  18467. DECLARE_BITMAP(pending_map, WHEEL_SIZE);
  18468. struct hlist_head vectors[WHEEL_SIZE];
  18469. + struct hlist_head expired_lists[LVL_DEPTH];
  18470. + int expired_count;
  18471. } ____cacheline_aligned;
  18472. static DEFINE_PER_CPU(struct timer_base, timer_bases[NR_BASES]);
  18473. @@ -948,10 +953,10 @@
  18474. if (!(tf & TIMER_MIGRATING)) {
  18475. base = get_timer_base(tf);
  18476. - spin_lock_irqsave(&base->lock, *flags);
  18477. + raw_spin_lock_irqsave(&base->lock, *flags);
  18478. if (timer->flags == tf)
  18479. return base;
  18480. - spin_unlock_irqrestore(&base->lock, *flags);
  18481. + raw_spin_unlock_irqrestore(&base->lock, *flags);
  18482. }
  18483. cpu_relax();
  18484. }
  18485. @@ -1023,9 +1028,9 @@
  18486. /* See the comment in lock_timer_base() */
  18487. timer->flags |= TIMER_MIGRATING;
  18488. - spin_unlock(&base->lock);
  18489. + raw_spin_unlock(&base->lock);
  18490. base = new_base;
  18491. - spin_lock(&base->lock);
  18492. + raw_spin_lock(&base->lock);
  18493. WRITE_ONCE(timer->flags,
  18494. (timer->flags & ~TIMER_BASEMASK) | base->cpu);
  18495. }
  18496. @@ -1050,7 +1055,7 @@
  18497. }
  18498. out_unlock:
  18499. - spin_unlock_irqrestore(&base->lock, flags);
  18500. + raw_spin_unlock_irqrestore(&base->lock, flags);
  18501. return ret;
  18502. }
  18503. @@ -1144,19 +1149,46 @@
  18504. if (base != new_base) {
  18505. timer->flags |= TIMER_MIGRATING;
  18506. - spin_unlock(&base->lock);
  18507. + raw_spin_unlock(&base->lock);
  18508. base = new_base;
  18509. - spin_lock(&base->lock);
  18510. + raw_spin_lock(&base->lock);
  18511. WRITE_ONCE(timer->flags,
  18512. (timer->flags & ~TIMER_BASEMASK) | cpu);
  18513. }
  18514. debug_activate(timer, timer->expires);
  18515. internal_add_timer(base, timer);
  18516. - spin_unlock_irqrestore(&base->lock, flags);
  18517. + raw_spin_unlock_irqrestore(&base->lock, flags);
  18518. }
  18519. EXPORT_SYMBOL_GPL(add_timer_on);
  18520. +#ifdef CONFIG_PREEMPT_RT_FULL
  18521. +/*
  18522. + * Wait for a running timer
  18523. + */
  18524. +static void wait_for_running_timer(struct timer_list *timer)
  18525. +{
  18526. + struct timer_base *base;
  18527. + u32 tf = timer->flags;
  18528. +
  18529. + if (tf & TIMER_MIGRATING)
  18530. + return;
  18531. +
  18532. + base = get_timer_base(tf);
  18533. + swait_event(base->wait_for_running_timer,
  18534. + base->running_timer != timer);
  18535. +}
  18536. +
  18537. +# define wakeup_timer_waiters(b) swake_up_all(&(b)->wait_for_running_timer)
  18538. +#else
  18539. +static inline void wait_for_running_timer(struct timer_list *timer)
  18540. +{
  18541. + cpu_relax();
  18542. +}
  18543. +
  18544. +# define wakeup_timer_waiters(b) do { } while (0)
  18545. +#endif
  18546. +
  18547. /**
  18548. * del_timer - deactive a timer.
  18549. * @timer: the timer to be deactivated
  18550. @@ -1180,7 +1212,7 @@
  18551. if (timer_pending(timer)) {
  18552. base = lock_timer_base(timer, &flags);
  18553. ret = detach_if_pending(timer, base, true);
  18554. - spin_unlock_irqrestore(&base->lock, flags);
  18555. + raw_spin_unlock_irqrestore(&base->lock, flags);
  18556. }
  18557. return ret;
  18558. @@ -1208,13 +1240,13 @@
  18559. timer_stats_timer_clear_start_info(timer);
  18560. ret = detach_if_pending(timer, base, true);
  18561. }
  18562. - spin_unlock_irqrestore(&base->lock, flags);
  18563. + raw_spin_unlock_irqrestore(&base->lock, flags);
  18564. return ret;
  18565. }
  18566. EXPORT_SYMBOL(try_to_del_timer_sync);
  18567. -#ifdef CONFIG_SMP
  18568. +#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
  18569. /**
  18570. * del_timer_sync - deactivate a timer and wait for the handler to finish.
  18571. * @timer: the timer to be deactivated
  18572. @@ -1274,7 +1306,7 @@
  18573. int ret = try_to_del_timer_sync(timer);
  18574. if (ret >= 0)
  18575. return ret;
  18576. - cpu_relax();
  18577. + wait_for_running_timer(timer);
  18578. }
  18579. }
  18580. EXPORT_SYMBOL(del_timer_sync);
  18581. @@ -1323,7 +1355,8 @@
  18582. }
  18583. }
  18584. -static void expire_timers(struct timer_base *base, struct hlist_head *head)
  18585. +static inline void __expire_timers(struct timer_base *base,
  18586. + struct hlist_head *head)
  18587. {
  18588. while (!hlist_empty(head)) {
  18589. struct timer_list *timer;
  18590. @@ -1339,33 +1372,53 @@
  18591. fn = timer->function;
  18592. data = timer->data;
  18593. - if (timer->flags & TIMER_IRQSAFE) {
  18594. - spin_unlock(&base->lock);
  18595. + if (!IS_ENABLED(CONFIG_PREEMPT_RT_FULL) &&
  18596. + timer->flags & TIMER_IRQSAFE) {
  18597. + raw_spin_unlock(&base->lock);
  18598. call_timer_fn(timer, fn, data);
  18599. - spin_lock(&base->lock);
  18600. + base->running_timer = NULL;
  18601. + raw_spin_lock(&base->lock);
  18602. } else {
  18603. - spin_unlock_irq(&base->lock);
  18604. + raw_spin_unlock_irq(&base->lock);
  18605. call_timer_fn(timer, fn, data);
  18606. - spin_lock_irq(&base->lock);
  18607. + base->running_timer = NULL;
  18608. + raw_spin_lock_irq(&base->lock);
  18609. }
  18610. }
  18611. }
  18612. -static int __collect_expired_timers(struct timer_base *base,
  18613. - struct hlist_head *heads)
  18614. +static void expire_timers(struct timer_base *base)
  18615. +{
  18616. + struct hlist_head *head;
  18617. +
  18618. + while (base->expired_count--) {
  18619. + head = base->expired_lists + base->expired_count;
  18620. + __expire_timers(base, head);
  18621. + }
  18622. + base->expired_count = 0;
  18623. +}
  18624. +
  18625. +static void __collect_expired_timers(struct timer_base *base)
  18626. {
  18627. unsigned long clk = base->clk;
  18628. struct hlist_head *vec;
  18629. - int i, levels = 0;
  18630. + int i;
  18631. unsigned int idx;
  18632. + /*
  18633. + * expire_timers() must be called at least once before we can
  18634. + * collect more timers
  18635. + */
  18636. + if (WARN_ON(base->expired_count))
  18637. + return;
  18638. +
  18639. for (i = 0; i < LVL_DEPTH; i++) {
  18640. idx = (clk & LVL_MASK) + i * LVL_SIZE;
  18641. if (__test_and_clear_bit(idx, base->pending_map)) {
  18642. vec = base->vectors + idx;
  18643. - hlist_move_list(vec, heads++);
  18644. - levels++;
  18645. + hlist_move_list(vec,
  18646. + &base->expired_lists[base->expired_count++]);
  18647. }
  18648. /* Is it time to look at the next level? */
  18649. if (clk & LVL_CLK_MASK)
  18650. @@ -1373,7 +1426,6 @@
  18651. /* Shift clock for the next level granularity */
  18652. clk >>= LVL_CLK_SHIFT;
  18653. }
  18654. - return levels;
  18655. }
  18656. #ifdef CONFIG_NO_HZ_COMMON
  18657. @@ -1515,7 +1567,7 @@
  18658. if (cpu_is_offline(smp_processor_id()))
  18659. return expires;
  18660. - spin_lock(&base->lock);
  18661. + raw_spin_lock(&base->lock);
  18662. nextevt = __next_timer_interrupt(base);
  18663. is_max_delta = (nextevt == base->clk + NEXT_TIMER_MAX_DELTA);
  18664. base->next_expiry = nextevt;
  18665. @@ -1543,7 +1595,7 @@
  18666. if ((expires - basem) > TICK_NSEC)
  18667. base->is_idle = true;
  18668. }
  18669. - spin_unlock(&base->lock);
  18670. + raw_spin_unlock(&base->lock);
  18671. return cmp_next_hrtimer_event(basem, expires);
  18672. }
  18673. @@ -1566,8 +1618,7 @@
  18674. base->is_idle = false;
  18675. }
  18676. -static int collect_expired_timers(struct timer_base *base,
  18677. - struct hlist_head *heads)
  18678. +static void collect_expired_timers(struct timer_base *base)
  18679. {
  18680. /*
  18681. * NOHZ optimization. After a long idle sleep we need to forward the
  18682. @@ -1584,20 +1635,49 @@
  18683. if (time_after(next, jiffies)) {
  18684. /* The call site will increment clock! */
  18685. base->clk = jiffies - 1;
  18686. - return 0;
  18687. + return;
  18688. }
  18689. base->clk = next;
  18690. }
  18691. - return __collect_expired_timers(base, heads);
  18692. + __collect_expired_timers(base);
  18693. }
  18694. #else
  18695. -static inline int collect_expired_timers(struct timer_base *base,
  18696. - struct hlist_head *heads)
  18697. +static inline void collect_expired_timers(struct timer_base *base)
  18698. {
  18699. - return __collect_expired_timers(base, heads);
  18700. + __collect_expired_timers(base);
  18701. }
  18702. #endif
  18703. +static int find_expired_timers(struct timer_base *base)
  18704. +{
  18705. + const unsigned long int end_clk = jiffies;
  18706. +
  18707. + while (!base->expired_count && time_after_eq(end_clk, base->clk)) {
  18708. + collect_expired_timers(base);
  18709. + base->clk++;
  18710. + }
  18711. +
  18712. + return base->expired_count;
  18713. +}
  18714. +
  18715. +/* Called from CPU tick routine to quickly collect expired timers */
  18716. +static int tick_find_expired(struct timer_base *base)
  18717. +{
  18718. + int count;
  18719. +
  18720. + raw_spin_lock(&base->lock);
  18721. +
  18722. + if (unlikely(time_after(jiffies, base->clk + HZ))) {
  18723. + /* defer to ktimersoftd; don't spend too long in irq context */
  18724. + count = -1;
  18725. + } else
  18726. + count = find_expired_timers(base);
  18727. +
  18728. + raw_spin_unlock(&base->lock);
  18729. +
  18730. + return count;
  18731. +}
  18732. +
  18733. /*
  18734. * Called from the timer interrupt handler to charge one tick to the current
  18735. * process. user_tick is 1 if the tick is user time, 0 for system.
  18736. @@ -1608,13 +1688,13 @@
  18737. /* Note: this timer irq context must be accounted for as well. */
  18738. account_process_tick(p, user_tick);
  18739. + scheduler_tick();
  18740. run_local_timers();
  18741. rcu_check_callbacks(user_tick);
  18742. -#ifdef CONFIG_IRQ_WORK
  18743. +#if defined(CONFIG_IRQ_WORK)
  18744. if (in_irq())
  18745. irq_work_tick();
  18746. #endif
  18747. - scheduler_tick();
  18748. run_posix_cpu_timers(p);
  18749. }
  18750. @@ -1624,24 +1704,13 @@
  18751. */
  18752. static inline void __run_timers(struct timer_base *base)
  18753. {
  18754. - struct hlist_head heads[LVL_DEPTH];
  18755. - int levels;
  18756. -
  18757. - if (!time_after_eq(jiffies, base->clk))
  18758. - return;
  18759. + raw_spin_lock_irq(&base->lock);
  18760. - spin_lock_irq(&base->lock);
  18761. + while (find_expired_timers(base))
  18762. + expire_timers(base);
  18763. - while (time_after_eq(jiffies, base->clk)) {
  18764. -
  18765. - levels = collect_expired_timers(base, heads);
  18766. - base->clk++;
  18767. -
  18768. - while (levels--)
  18769. - expire_timers(base, heads + levels);
  18770. - }
  18771. - base->running_timer = NULL;
  18772. - spin_unlock_irq(&base->lock);
  18773. + raw_spin_unlock_irq(&base->lock);
  18774. + wakeup_timer_waiters(base);
  18775. }
  18776. /*
  18777. @@ -1651,6 +1720,8 @@
  18778. {
  18779. struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
  18780. + irq_work_tick_soft();
  18781. +
  18782. __run_timers(base);
  18783. if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && base->nohz_active)
  18784. __run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));
  18785. @@ -1665,12 +1736,12 @@
  18786. hrtimer_run_queues();
  18787. /* Raise the softirq only if required. */
  18788. - if (time_before(jiffies, base->clk)) {
  18789. + if (time_before(jiffies, base->clk) || !tick_find_expired(base)) {
  18790. if (!IS_ENABLED(CONFIG_NO_HZ_COMMON) || !base->nohz_active)
  18791. return;
  18792. /* CPU is awake, so check the deferrable base. */
  18793. base++;
  18794. - if (time_before(jiffies, base->clk))
  18795. + if (time_before(jiffies, base->clk) || !tick_find_expired(base))
  18796. return;
  18797. }
  18798. raise_softirq(TIMER_SOFTIRQ);
  18799. @@ -1836,16 +1907,17 @@
  18800. * The caller is globally serialized and nobody else
  18801. * takes two locks at once, deadlock is not possible.
  18802. */
  18803. - spin_lock_irq(&new_base->lock);
  18804. - spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
  18805. + raw_spin_lock_irq(&new_base->lock);
  18806. + raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
  18807. BUG_ON(old_base->running_timer);
  18808. + BUG_ON(old_base->expired_count);
  18809. for (i = 0; i < WHEEL_SIZE; i++)
  18810. migrate_timer_list(new_base, old_base->vectors + i);
  18811. - spin_unlock(&old_base->lock);
  18812. - spin_unlock_irq(&new_base->lock);
  18813. + raw_spin_unlock(&old_base->lock);
  18814. + raw_spin_unlock_irq(&new_base->lock);
  18815. put_cpu_ptr(&timer_bases);
  18816. }
  18817. return 0;
  18818. @@ -1861,8 +1933,12 @@
  18819. for (i = 0; i < NR_BASES; i++) {
  18820. base = per_cpu_ptr(&timer_bases[i], cpu);
  18821. base->cpu = cpu;
  18822. - spin_lock_init(&base->lock);
  18823. + raw_spin_lock_init(&base->lock);
  18824. base->clk = jiffies;
  18825. +#ifdef CONFIG_PREEMPT_RT_FULL
  18826. + init_swait_queue_head(&base->wait_for_running_timer);
  18827. +#endif
  18828. + base->expired_count = 0;
  18829. }
  18830. }
  18831. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/Kconfig
  18832. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/Kconfig 2017-04-16 10:38:30.000000000 +0200
  18833. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/Kconfig 2017-04-18 17:54:26.000000000 +0200
  18834. @@ -182,6 +182,24 @@
  18835. enabled. This option and the preempt-off timing option can be
  18836. used together or separately.)
  18837. +config INTERRUPT_OFF_HIST
  18838. + bool "Interrupts-off Latency Histogram"
  18839. + depends on IRQSOFF_TRACER
  18840. + help
  18841. + This option generates continuously updated histograms (one per cpu)
  18842. + of the duration of time periods with interrupts disabled. The
  18843. + histograms are disabled by default. To enable them, write a non-zero
  18844. + number to
  18845. +
  18846. + /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff
  18847. +
  18848. + If PREEMPT_OFF_HIST is also selected, additional histograms (one
  18849. + per cpu) are generated that accumulate the duration of time periods
  18850. + when both interrupts and preemption are disabled. The histogram data
  18851. + will be located in the debug file system at
  18852. +
  18853. + /sys/kernel/debug/tracing/latency_hist/irqsoff
  18854. +
  18855. config PREEMPT_TRACER
  18856. bool "Preemption-off Latency Tracer"
  18857. default n
  18858. @@ -206,6 +224,24 @@
  18859. enabled. This option and the irqs-off timing option can be
  18860. used together or separately.)
  18861. +config PREEMPT_OFF_HIST
  18862. + bool "Preemption-off Latency Histogram"
  18863. + depends on PREEMPT_TRACER
  18864. + help
  18865. + This option generates continuously updated histograms (one per cpu)
  18866. + of the duration of time periods with preemption disabled. The
  18867. + histograms are disabled by default. To enable them, write a non-zero
  18868. + number to
  18869. +
  18870. + /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff
  18871. +
  18872. + If INTERRUPT_OFF_HIST is also selected, additional histograms (one
  18873. + per cpu) are generated that accumulate the duration of time periods
  18874. + when both interrupts and preemption are disabled. The histogram data
  18875. + will be located in the debug file system at
  18876. +
  18877. + /sys/kernel/debug/tracing/latency_hist/preemptoff
  18878. +
  18879. config SCHED_TRACER
  18880. bool "Scheduling Latency Tracer"
  18881. select GENERIC_TRACER
  18882. @@ -251,6 +287,74 @@
  18883. file. Every time a latency is greater than tracing_thresh, it will
  18884. be recorded into the ring buffer.
  18885. +config WAKEUP_LATENCY_HIST
  18886. + bool "Scheduling Latency Histogram"
  18887. + depends on SCHED_TRACER
  18888. + help
  18889. + This option generates continuously updated histograms (one per cpu)
  18890. + of the scheduling latency of the highest priority task.
  18891. + The histograms are disabled by default. To enable them, write a
  18892. + non-zero number to
  18893. +
  18894. + /sys/kernel/debug/tracing/latency_hist/enable/wakeup
  18895. +
  18896. + Two different algorithms are used, one to determine the latency of
  18897. + processes that exclusively use the highest priority of the system and
  18898. + another one to determine the latency of processes that share the
  18899. + highest system priority with other processes. The former is used to
  18900. + improve hardware and system software, the latter to optimize the
  18901. + priority design of a given system. The histogram data will be
  18902. + located in the debug file system at
  18903. +
  18904. + /sys/kernel/debug/tracing/latency_hist/wakeup
  18905. +
  18906. + and
  18907. +
  18908. + /sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio
  18909. +
  18910. + If both Scheduling Latency Histogram and Missed Timer Offsets
  18911. + Histogram are selected, additional histogram data will be collected
  18912. + that contain, in addition to the wakeup latency, the timer latency, in
  18913. + case the wakeup was triggered by an expired timer. These histograms
  18914. + are available in the
  18915. +
  18916. + /sys/kernel/debug/tracing/latency_hist/timerandwakeup
  18917. +
  18918. + directory. They reflect the apparent interrupt and scheduling latency
  18919. + and are best suitable to determine the worst-case latency of a given
  18920. + system. To enable these histograms, write a non-zero number to
  18921. +
  18922. + /sys/kernel/debug/tracing/latency_hist/enable/timerandwakeup
  18923. +
  18924. +config MISSED_TIMER_OFFSETS_HIST
  18925. + depends on HIGH_RES_TIMERS
  18926. + select GENERIC_TRACER
  18927. + bool "Missed Timer Offsets Histogram"
  18928. + help
  18929. + Generate a histogram of missed timer offsets in microseconds. The
  18930. + histograms are disabled by default. To enable them, write a non-zero
  18931. + number to
  18932. +
  18933. + /sys/kernel/debug/tracing/latency_hist/enable/missed_timer_offsets
  18934. +
  18935. + The histogram data will be located in the debug file system at
  18936. +
  18937. + /sys/kernel/debug/tracing/latency_hist/missed_timer_offsets
  18938. +
  18939. + If both Scheduling Latency Histogram and Missed Timer Offsets
  18940. + Histogram are selected, additional histogram data will be collected
  18941. + that contain, in addition to the wakeup latency, the timer latency, in
  18942. + case the wakeup was triggered by an expired timer. These histograms
  18943. + are available in the
  18944. +
  18945. + /sys/kernel/debug/tracing/latency_hist/timerandwakeup
  18946. +
  18947. + directory. They reflect the apparent interrupt and scheduling latency
  18948. + and are best suitable to determine the worst-case latency of a given
  18949. + system. To enable these histograms, write a non-zero number to
  18950. +
  18951. + /sys/kernel/debug/tracing/latency_hist/enable/timerandwakeup
  18952. +
  18953. config ENABLE_DEFAULT_TRACERS
  18954. bool "Trace process context switches and events"
  18955. depends on !GENERIC_TRACER
  18956. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/Makefile linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/Makefile
  18957. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/Makefile 2017-04-16 10:38:30.000000000 +0200
  18958. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/Makefile 2017-04-18 17:54:26.000000000 +0200
  18959. @@ -38,6 +38,10 @@
  18960. obj-$(CONFIG_PREEMPT_TRACER) += trace_irqsoff.o
  18961. obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o
  18962. obj-$(CONFIG_HWLAT_TRACER) += trace_hwlat.o
  18963. +obj-$(CONFIG_INTERRUPT_OFF_HIST) += latency_hist.o
  18964. +obj-$(CONFIG_PREEMPT_OFF_HIST) += latency_hist.o
  18965. +obj-$(CONFIG_WAKEUP_LATENCY_HIST) += latency_hist.o
  18966. +obj-$(CONFIG_MISSED_TIMER_OFFSETS_HIST) += latency_hist.o
  18967. obj-$(CONFIG_NOP_TRACER) += trace_nop.o
  18968. obj-$(CONFIG_STACK_TRACER) += trace_stack.o
  18969. obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o
  18970. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/latency_hist.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/latency_hist.c
  18971. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/latency_hist.c 1970-01-01 01:00:00.000000000 +0100
  18972. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/latency_hist.c 2017-04-18 17:54:26.000000000 +0200
  18973. @@ -0,0 +1,1178 @@
  18974. +/*
  18975. + * kernel/trace/latency_hist.c
  18976. + *
  18977. + * Add support for histograms of preemption-off latency and
  18978. + * interrupt-off latency and wakeup latency, it depends on
  18979. + * Real-Time Preemption Support.
  18980. + *
  18981. + * Copyright (C) 2005 MontaVista Software, Inc.
  18982. + * Yi Yang <yyang@ch.mvista.com>
  18983. + *
  18984. + * Converted to work with the new latency tracer.
  18985. + * Copyright (C) 2008 Red Hat, Inc.
  18986. + * Steven Rostedt <srostedt@redhat.com>
  18987. + *
  18988. + */
  18989. +#include <linux/module.h>
  18990. +#include <linux/debugfs.h>
  18991. +#include <linux/seq_file.h>
  18992. +#include <linux/percpu.h>
  18993. +#include <linux/kallsyms.h>
  18994. +#include <linux/uaccess.h>
  18995. +#include <linux/sched.h>
  18996. +#include <linux/sched/rt.h>
  18997. +#include <linux/slab.h>
  18998. +#include <linux/atomic.h>
  18999. +#include <asm/div64.h>
  19000. +
  19001. +#include "trace.h"
  19002. +#include <trace/events/sched.h>
  19003. +
  19004. +#define NSECS_PER_USECS 1000L
  19005. +
  19006. +#define CREATE_TRACE_POINTS
  19007. +#include <trace/events/hist.h>
  19008. +
  19009. +enum {
  19010. + IRQSOFF_LATENCY = 0,
  19011. + PREEMPTOFF_LATENCY,
  19012. + PREEMPTIRQSOFF_LATENCY,
  19013. + WAKEUP_LATENCY,
  19014. + WAKEUP_LATENCY_SHAREDPRIO,
  19015. + MISSED_TIMER_OFFSETS,
  19016. + TIMERANDWAKEUP_LATENCY,
  19017. + MAX_LATENCY_TYPE,
  19018. +};
  19019. +
  19020. +#define MAX_ENTRY_NUM 10240
  19021. +
  19022. +struct hist_data {
  19023. + atomic_t hist_mode; /* 0 log, 1 don't log */
  19024. + long offset; /* set it to MAX_ENTRY_NUM/2 for a bipolar scale */
  19025. + long min_lat;
  19026. + long max_lat;
  19027. + unsigned long long below_hist_bound_samples;
  19028. + unsigned long long above_hist_bound_samples;
  19029. + long long accumulate_lat;
  19030. + unsigned long long total_samples;
  19031. + unsigned long long hist_array[MAX_ENTRY_NUM];
  19032. +};
  19033. +
  19034. +struct enable_data {
  19035. + int latency_type;
  19036. + int enabled;
  19037. +};
  19038. +
  19039. +static char *latency_hist_dir_root = "latency_hist";
  19040. +
  19041. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  19042. +static DEFINE_PER_CPU(struct hist_data, irqsoff_hist);
  19043. +static char *irqsoff_hist_dir = "irqsoff";
  19044. +static DEFINE_PER_CPU(cycles_t, hist_irqsoff_start);
  19045. +static DEFINE_PER_CPU(int, hist_irqsoff_counting);
  19046. +#endif
  19047. +
  19048. +#ifdef CONFIG_PREEMPT_OFF_HIST
  19049. +static DEFINE_PER_CPU(struct hist_data, preemptoff_hist);
  19050. +static char *preemptoff_hist_dir = "preemptoff";
  19051. +static DEFINE_PER_CPU(cycles_t, hist_preemptoff_start);
  19052. +static DEFINE_PER_CPU(int, hist_preemptoff_counting);
  19053. +#endif
  19054. +
  19055. +#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST)
  19056. +static DEFINE_PER_CPU(struct hist_data, preemptirqsoff_hist);
  19057. +static char *preemptirqsoff_hist_dir = "preemptirqsoff";
  19058. +static DEFINE_PER_CPU(cycles_t, hist_preemptirqsoff_start);
  19059. +static DEFINE_PER_CPU(int, hist_preemptirqsoff_counting);
  19060. +#endif
  19061. +
  19062. +#if defined(CONFIG_PREEMPT_OFF_HIST) || defined(CONFIG_INTERRUPT_OFF_HIST)
  19063. +static notrace void probe_preemptirqsoff_hist(void *v, int reason, int start);
  19064. +static struct enable_data preemptirqsoff_enabled_data = {
  19065. + .latency_type = PREEMPTIRQSOFF_LATENCY,
  19066. + .enabled = 0,
  19067. +};
  19068. +#endif
  19069. +
  19070. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  19071. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19072. +struct maxlatproc_data {
  19073. + char comm[FIELD_SIZEOF(struct task_struct, comm)];
  19074. + char current_comm[FIELD_SIZEOF(struct task_struct, comm)];
  19075. + int pid;
  19076. + int current_pid;
  19077. + int prio;
  19078. + int current_prio;
  19079. + long latency;
  19080. + long timeroffset;
  19081. + cycle_t timestamp;
  19082. +};
  19083. +#endif
  19084. +
  19085. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  19086. +static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist);
  19087. +static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist_sharedprio);
  19088. +static char *wakeup_latency_hist_dir = "wakeup";
  19089. +static char *wakeup_latency_hist_dir_sharedprio = "sharedprio";
  19090. +static notrace void probe_wakeup_latency_hist_start(void *v,
  19091. + struct task_struct *p);
  19092. +static notrace void probe_wakeup_latency_hist_stop(void *v,
  19093. + bool preempt, struct task_struct *prev, struct task_struct *next);
  19094. +static notrace void probe_sched_migrate_task(void *,
  19095. + struct task_struct *task, int cpu);
  19096. +static struct enable_data wakeup_latency_enabled_data = {
  19097. + .latency_type = WAKEUP_LATENCY,
  19098. + .enabled = 0,
  19099. +};
  19100. +static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc);
  19101. +static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc_sharedprio);
  19102. +static DEFINE_PER_CPU(struct task_struct *, wakeup_task);
  19103. +static DEFINE_PER_CPU(int, wakeup_sharedprio);
  19104. +static unsigned long wakeup_pid;
  19105. +#endif
  19106. +
  19107. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  19108. +static DEFINE_PER_CPU(struct hist_data, missed_timer_offsets);
  19109. +static char *missed_timer_offsets_dir = "missed_timer_offsets";
  19110. +static notrace void probe_hrtimer_interrupt(void *v, int cpu,
  19111. + long long offset, struct task_struct *curr, struct task_struct *task);
  19112. +static struct enable_data missed_timer_offsets_enabled_data = {
  19113. + .latency_type = MISSED_TIMER_OFFSETS,
  19114. + .enabled = 0,
  19115. +};
  19116. +static DEFINE_PER_CPU(struct maxlatproc_data, missed_timer_offsets_maxlatproc);
  19117. +static unsigned long missed_timer_offsets_pid;
  19118. +#endif
  19119. +
  19120. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \
  19121. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19122. +static DEFINE_PER_CPU(struct hist_data, timerandwakeup_latency_hist);
  19123. +static char *timerandwakeup_latency_hist_dir = "timerandwakeup";
  19124. +static struct enable_data timerandwakeup_enabled_data = {
  19125. + .latency_type = TIMERANDWAKEUP_LATENCY,
  19126. + .enabled = 0,
  19127. +};
  19128. +static DEFINE_PER_CPU(struct maxlatproc_data, timerandwakeup_maxlatproc);
  19129. +#endif
  19130. +
  19131. +void notrace latency_hist(int latency_type, int cpu, long latency,
  19132. + long timeroffset, cycle_t stop,
  19133. + struct task_struct *p)
  19134. +{
  19135. + struct hist_data *my_hist;
  19136. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  19137. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19138. + struct maxlatproc_data *mp = NULL;
  19139. +#endif
  19140. +
  19141. + if (!cpu_possible(cpu) || latency_type < 0 ||
  19142. + latency_type >= MAX_LATENCY_TYPE)
  19143. + return;
  19144. +
  19145. + switch (latency_type) {
  19146. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  19147. + case IRQSOFF_LATENCY:
  19148. + my_hist = &per_cpu(irqsoff_hist, cpu);
  19149. + break;
  19150. +#endif
  19151. +#ifdef CONFIG_PREEMPT_OFF_HIST
  19152. + case PREEMPTOFF_LATENCY:
  19153. + my_hist = &per_cpu(preemptoff_hist, cpu);
  19154. + break;
  19155. +#endif
  19156. +#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST)
  19157. + case PREEMPTIRQSOFF_LATENCY:
  19158. + my_hist = &per_cpu(preemptirqsoff_hist, cpu);
  19159. + break;
  19160. +#endif
  19161. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  19162. + case WAKEUP_LATENCY:
  19163. + my_hist = &per_cpu(wakeup_latency_hist, cpu);
  19164. + mp = &per_cpu(wakeup_maxlatproc, cpu);
  19165. + break;
  19166. + case WAKEUP_LATENCY_SHAREDPRIO:
  19167. + my_hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu);
  19168. + mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu);
  19169. + break;
  19170. +#endif
  19171. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  19172. + case MISSED_TIMER_OFFSETS:
  19173. + my_hist = &per_cpu(missed_timer_offsets, cpu);
  19174. + mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu);
  19175. + break;
  19176. +#endif
  19177. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \
  19178. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19179. + case TIMERANDWAKEUP_LATENCY:
  19180. + my_hist = &per_cpu(timerandwakeup_latency_hist, cpu);
  19181. + mp = &per_cpu(timerandwakeup_maxlatproc, cpu);
  19182. + break;
  19183. +#endif
  19184. +
  19185. + default:
  19186. + return;
  19187. + }
  19188. +
  19189. + latency += my_hist->offset;
  19190. +
  19191. + if (atomic_read(&my_hist->hist_mode) == 0)
  19192. + return;
  19193. +
  19194. + if (latency < 0 || latency >= MAX_ENTRY_NUM) {
  19195. + if (latency < 0)
  19196. + my_hist->below_hist_bound_samples++;
  19197. + else
  19198. + my_hist->above_hist_bound_samples++;
  19199. + } else
  19200. + my_hist->hist_array[latency]++;
  19201. +
  19202. + if (unlikely(latency > my_hist->max_lat ||
  19203. + my_hist->min_lat == LONG_MAX)) {
  19204. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  19205. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19206. + if (latency_type == WAKEUP_LATENCY ||
  19207. + latency_type == WAKEUP_LATENCY_SHAREDPRIO ||
  19208. + latency_type == MISSED_TIMER_OFFSETS ||
  19209. + latency_type == TIMERANDWAKEUP_LATENCY) {
  19210. + strncpy(mp->comm, p->comm, sizeof(mp->comm));
  19211. + strncpy(mp->current_comm, current->comm,
  19212. + sizeof(mp->current_comm));
  19213. + mp->pid = task_pid_nr(p);
  19214. + mp->current_pid = task_pid_nr(current);
  19215. + mp->prio = p->prio;
  19216. + mp->current_prio = current->prio;
  19217. + mp->latency = latency;
  19218. + mp->timeroffset = timeroffset;
  19219. + mp->timestamp = stop;
  19220. + }
  19221. +#endif
  19222. + my_hist->max_lat = latency;
  19223. + }
  19224. + if (unlikely(latency < my_hist->min_lat))
  19225. + my_hist->min_lat = latency;
  19226. + my_hist->total_samples++;
  19227. + my_hist->accumulate_lat += latency;
  19228. +}
  19229. +
  19230. +static void *l_start(struct seq_file *m, loff_t *pos)
  19231. +{
  19232. + loff_t *index_ptr = NULL;
  19233. + loff_t index = *pos;
  19234. + struct hist_data *my_hist = m->private;
  19235. +
  19236. + if (index == 0) {
  19237. + char minstr[32], avgstr[32], maxstr[32];
  19238. +
  19239. + atomic_dec(&my_hist->hist_mode);
  19240. +
  19241. + if (likely(my_hist->total_samples)) {
  19242. + long avg = (long) div64_s64(my_hist->accumulate_lat,
  19243. + my_hist->total_samples);
  19244. + snprintf(minstr, sizeof(minstr), "%ld",
  19245. + my_hist->min_lat - my_hist->offset);
  19246. + snprintf(avgstr, sizeof(avgstr), "%ld",
  19247. + avg - my_hist->offset);
  19248. + snprintf(maxstr, sizeof(maxstr), "%ld",
  19249. + my_hist->max_lat - my_hist->offset);
  19250. + } else {
  19251. + strcpy(minstr, "<undef>");
  19252. + strcpy(avgstr, minstr);
  19253. + strcpy(maxstr, minstr);
  19254. + }
  19255. +
  19256. + seq_printf(m, "#Minimum latency: %s microseconds\n"
  19257. + "#Average latency: %s microseconds\n"
  19258. + "#Maximum latency: %s microseconds\n"
  19259. + "#Total samples: %llu\n"
  19260. + "#There are %llu samples lower than %ld"
  19261. + " microseconds.\n"
  19262. + "#There are %llu samples greater or equal"
  19263. + " than %ld microseconds.\n"
  19264. + "#usecs\t%16s\n",
  19265. + minstr, avgstr, maxstr,
  19266. + my_hist->total_samples,
  19267. + my_hist->below_hist_bound_samples,
  19268. + -my_hist->offset,
  19269. + my_hist->above_hist_bound_samples,
  19270. + MAX_ENTRY_NUM - my_hist->offset,
  19271. + "samples");
  19272. + }
  19273. + if (index < MAX_ENTRY_NUM) {
  19274. + index_ptr = kmalloc(sizeof(loff_t), GFP_KERNEL);
  19275. + if (index_ptr)
  19276. + *index_ptr = index;
  19277. + }
  19278. +
  19279. + return index_ptr;
  19280. +}
  19281. +
  19282. +static void *l_next(struct seq_file *m, void *p, loff_t *pos)
  19283. +{
  19284. + loff_t *index_ptr = p;
  19285. + struct hist_data *my_hist = m->private;
  19286. +
  19287. + if (++*pos >= MAX_ENTRY_NUM) {
  19288. + atomic_inc(&my_hist->hist_mode);
  19289. + return NULL;
  19290. + }
  19291. + *index_ptr = *pos;
  19292. + return index_ptr;
  19293. +}
  19294. +
  19295. +static void l_stop(struct seq_file *m, void *p)
  19296. +{
  19297. + kfree(p);
  19298. +}
  19299. +
  19300. +static int l_show(struct seq_file *m, void *p)
  19301. +{
  19302. + int index = *(loff_t *) p;
  19303. + struct hist_data *my_hist = m->private;
  19304. +
  19305. + seq_printf(m, "%6ld\t%16llu\n", index - my_hist->offset,
  19306. + my_hist->hist_array[index]);
  19307. + return 0;
  19308. +}
  19309. +
  19310. +static const struct seq_operations latency_hist_seq_op = {
  19311. + .start = l_start,
  19312. + .next = l_next,
  19313. + .stop = l_stop,
  19314. + .show = l_show
  19315. +};
  19316. +
  19317. +static int latency_hist_open(struct inode *inode, struct file *file)
  19318. +{
  19319. + int ret;
  19320. +
  19321. + ret = seq_open(file, &latency_hist_seq_op);
  19322. + if (!ret) {
  19323. + struct seq_file *seq = file->private_data;
  19324. + seq->private = inode->i_private;
  19325. + }
  19326. + return ret;
  19327. +}
  19328. +
  19329. +static const struct file_operations latency_hist_fops = {
  19330. + .open = latency_hist_open,
  19331. + .read = seq_read,
  19332. + .llseek = seq_lseek,
  19333. + .release = seq_release,
  19334. +};
  19335. +
  19336. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  19337. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19338. +static void clear_maxlatprocdata(struct maxlatproc_data *mp)
  19339. +{
  19340. + mp->comm[0] = mp->current_comm[0] = '\0';
  19341. + mp->prio = mp->current_prio = mp->pid = mp->current_pid =
  19342. + mp->latency = mp->timeroffset = -1;
  19343. + mp->timestamp = 0;
  19344. +}
  19345. +#endif
  19346. +
  19347. +static void hist_reset(struct hist_data *hist)
  19348. +{
  19349. + atomic_dec(&hist->hist_mode);
  19350. +
  19351. + memset(hist->hist_array, 0, sizeof(hist->hist_array));
  19352. + hist->below_hist_bound_samples = 0ULL;
  19353. + hist->above_hist_bound_samples = 0ULL;
  19354. + hist->min_lat = LONG_MAX;
  19355. + hist->max_lat = LONG_MIN;
  19356. + hist->total_samples = 0ULL;
  19357. + hist->accumulate_lat = 0LL;
  19358. +
  19359. + atomic_inc(&hist->hist_mode);
  19360. +}
  19361. +
  19362. +static ssize_t
  19363. +latency_hist_reset(struct file *file, const char __user *a,
  19364. + size_t size, loff_t *off)
  19365. +{
  19366. + int cpu;
  19367. + struct hist_data *hist = NULL;
  19368. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  19369. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19370. + struct maxlatproc_data *mp = NULL;
  19371. +#endif
  19372. + off_t latency_type = (off_t) file->private_data;
  19373. +
  19374. + for_each_online_cpu(cpu) {
  19375. +
  19376. + switch (latency_type) {
  19377. +#ifdef CONFIG_PREEMPT_OFF_HIST
  19378. + case PREEMPTOFF_LATENCY:
  19379. + hist = &per_cpu(preemptoff_hist, cpu);
  19380. + break;
  19381. +#endif
  19382. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  19383. + case IRQSOFF_LATENCY:
  19384. + hist = &per_cpu(irqsoff_hist, cpu);
  19385. + break;
  19386. +#endif
  19387. +#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
  19388. + case PREEMPTIRQSOFF_LATENCY:
  19389. + hist = &per_cpu(preemptirqsoff_hist, cpu);
  19390. + break;
  19391. +#endif
  19392. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  19393. + case WAKEUP_LATENCY:
  19394. + hist = &per_cpu(wakeup_latency_hist, cpu);
  19395. + mp = &per_cpu(wakeup_maxlatproc, cpu);
  19396. + break;
  19397. + case WAKEUP_LATENCY_SHAREDPRIO:
  19398. + hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu);
  19399. + mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu);
  19400. + break;
  19401. +#endif
  19402. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  19403. + case MISSED_TIMER_OFFSETS:
  19404. + hist = &per_cpu(missed_timer_offsets, cpu);
  19405. + mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu);
  19406. + break;
  19407. +#endif
  19408. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \
  19409. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19410. + case TIMERANDWAKEUP_LATENCY:
  19411. + hist = &per_cpu(timerandwakeup_latency_hist, cpu);
  19412. + mp = &per_cpu(timerandwakeup_maxlatproc, cpu);
  19413. + break;
  19414. +#endif
  19415. + }
  19416. +
  19417. + hist_reset(hist);
  19418. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  19419. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19420. + if (latency_type == WAKEUP_LATENCY ||
  19421. + latency_type == WAKEUP_LATENCY_SHAREDPRIO ||
  19422. + latency_type == MISSED_TIMER_OFFSETS ||
  19423. + latency_type == TIMERANDWAKEUP_LATENCY)
  19424. + clear_maxlatprocdata(mp);
  19425. +#endif
  19426. + }
  19427. +
  19428. + return size;
  19429. +}
  19430. +
  19431. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  19432. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19433. +static ssize_t
  19434. +show_pid(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos)
  19435. +{
  19436. + char buf[64];
  19437. + int r;
  19438. + unsigned long *this_pid = file->private_data;
  19439. +
  19440. + r = snprintf(buf, sizeof(buf), "%lu\n", *this_pid);
  19441. + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  19442. +}
  19443. +
  19444. +static ssize_t do_pid(struct file *file, const char __user *ubuf,
  19445. + size_t cnt, loff_t *ppos)
  19446. +{
  19447. + char buf[64];
  19448. + unsigned long pid;
  19449. + unsigned long *this_pid = file->private_data;
  19450. +
  19451. + if (cnt >= sizeof(buf))
  19452. + return -EINVAL;
  19453. +
  19454. + if (copy_from_user(&buf, ubuf, cnt))
  19455. + return -EFAULT;
  19456. +
  19457. + buf[cnt] = '\0';
  19458. +
  19459. + if (kstrtoul(buf, 10, &pid))
  19460. + return -EINVAL;
  19461. +
  19462. + *this_pid = pid;
  19463. +
  19464. + return cnt;
  19465. +}
  19466. +#endif
  19467. +
  19468. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  19469. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19470. +static ssize_t
  19471. +show_maxlatproc(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos)
  19472. +{
  19473. + int r;
  19474. + struct maxlatproc_data *mp = file->private_data;
  19475. + int strmaxlen = (TASK_COMM_LEN * 2) + (8 * 8);
  19476. + unsigned long long t;
  19477. + unsigned long usecs, secs;
  19478. + char *buf;
  19479. +
  19480. + if (mp->pid == -1 || mp->current_pid == -1) {
  19481. + buf = "(none)\n";
  19482. + return simple_read_from_buffer(ubuf, cnt, ppos, buf,
  19483. + strlen(buf));
  19484. + }
  19485. +
  19486. + buf = kmalloc(strmaxlen, GFP_KERNEL);
  19487. + if (buf == NULL)
  19488. + return -ENOMEM;
  19489. +
  19490. + t = ns2usecs(mp->timestamp);
  19491. + usecs = do_div(t, USEC_PER_SEC);
  19492. + secs = (unsigned long) t;
  19493. + r = snprintf(buf, strmaxlen,
  19494. + "%d %d %ld (%ld) %s <- %d %d %s %lu.%06lu\n", mp->pid,
  19495. + MAX_RT_PRIO-1 - mp->prio, mp->latency, mp->timeroffset, mp->comm,
  19496. + mp->current_pid, MAX_RT_PRIO-1 - mp->current_prio, mp->current_comm,
  19497. + secs, usecs);
  19498. + r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  19499. + kfree(buf);
  19500. + return r;
  19501. +}
  19502. +#endif
  19503. +
  19504. +static ssize_t
  19505. +show_enable(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos)
  19506. +{
  19507. + char buf[64];
  19508. + struct enable_data *ed = file->private_data;
  19509. + int r;
  19510. +
  19511. + r = snprintf(buf, sizeof(buf), "%d\n", ed->enabled);
  19512. + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  19513. +}
  19514. +
  19515. +static ssize_t
  19516. +do_enable(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos)
  19517. +{
  19518. + char buf[64];
  19519. + long enable;
  19520. + struct enable_data *ed = file->private_data;
  19521. +
  19522. + if (cnt >= sizeof(buf))
  19523. + return -EINVAL;
  19524. +
  19525. + if (copy_from_user(&buf, ubuf, cnt))
  19526. + return -EFAULT;
  19527. +
  19528. + buf[cnt] = 0;
  19529. +
  19530. + if (kstrtoul(buf, 10, &enable))
  19531. + return -EINVAL;
  19532. +
  19533. + if ((enable && ed->enabled) || (!enable && !ed->enabled))
  19534. + return cnt;
  19535. +
  19536. + if (enable) {
  19537. + int ret;
  19538. +
  19539. + switch (ed->latency_type) {
  19540. +#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST)
  19541. + case PREEMPTIRQSOFF_LATENCY:
  19542. + ret = register_trace_preemptirqsoff_hist(
  19543. + probe_preemptirqsoff_hist, NULL);
  19544. + if (ret) {
  19545. + pr_info("wakeup trace: Couldn't assign "
  19546. + "probe_preemptirqsoff_hist "
  19547. + "to trace_preemptirqsoff_hist\n");
  19548. + return ret;
  19549. + }
  19550. + break;
  19551. +#endif
  19552. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  19553. + case WAKEUP_LATENCY:
  19554. + ret = register_trace_sched_wakeup(
  19555. + probe_wakeup_latency_hist_start, NULL);
  19556. + if (ret) {
  19557. + pr_info("wakeup trace: Couldn't assign "
  19558. + "probe_wakeup_latency_hist_start "
  19559. + "to trace_sched_wakeup\n");
  19560. + return ret;
  19561. + }
  19562. + ret = register_trace_sched_wakeup_new(
  19563. + probe_wakeup_latency_hist_start, NULL);
  19564. + if (ret) {
  19565. + pr_info("wakeup trace: Couldn't assign "
  19566. + "probe_wakeup_latency_hist_start "
  19567. + "to trace_sched_wakeup_new\n");
  19568. + unregister_trace_sched_wakeup(
  19569. + probe_wakeup_latency_hist_start, NULL);
  19570. + return ret;
  19571. + }
  19572. + ret = register_trace_sched_switch(
  19573. + probe_wakeup_latency_hist_stop, NULL);
  19574. + if (ret) {
  19575. + pr_info("wakeup trace: Couldn't assign "
  19576. + "probe_wakeup_latency_hist_stop "
  19577. + "to trace_sched_switch\n");
  19578. + unregister_trace_sched_wakeup(
  19579. + probe_wakeup_latency_hist_start, NULL);
  19580. + unregister_trace_sched_wakeup_new(
  19581. + probe_wakeup_latency_hist_start, NULL);
  19582. + return ret;
  19583. + }
  19584. + ret = register_trace_sched_migrate_task(
  19585. + probe_sched_migrate_task, NULL);
  19586. + if (ret) {
  19587. + pr_info("wakeup trace: Couldn't assign "
  19588. + "probe_sched_migrate_task "
  19589. + "to trace_sched_migrate_task\n");
  19590. + unregister_trace_sched_wakeup(
  19591. + probe_wakeup_latency_hist_start, NULL);
  19592. + unregister_trace_sched_wakeup_new(
  19593. + probe_wakeup_latency_hist_start, NULL);
  19594. + unregister_trace_sched_switch(
  19595. + probe_wakeup_latency_hist_stop, NULL);
  19596. + return ret;
  19597. + }
  19598. + break;
  19599. +#endif
  19600. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  19601. + case MISSED_TIMER_OFFSETS:
  19602. + ret = register_trace_hrtimer_interrupt(
  19603. + probe_hrtimer_interrupt, NULL);
  19604. + if (ret) {
  19605. + pr_info("wakeup trace: Couldn't assign "
  19606. + "probe_hrtimer_interrupt "
  19607. + "to trace_hrtimer_interrupt\n");
  19608. + return ret;
  19609. + }
  19610. + break;
  19611. +#endif
  19612. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \
  19613. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19614. + case TIMERANDWAKEUP_LATENCY:
  19615. + if (!wakeup_latency_enabled_data.enabled ||
  19616. + !missed_timer_offsets_enabled_data.enabled)
  19617. + return -EINVAL;
  19618. + break;
  19619. +#endif
  19620. + default:
  19621. + break;
  19622. + }
  19623. + } else {
  19624. + switch (ed->latency_type) {
  19625. +#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST)
  19626. + case PREEMPTIRQSOFF_LATENCY:
  19627. + {
  19628. + int cpu;
  19629. +
  19630. + unregister_trace_preemptirqsoff_hist(
  19631. + probe_preemptirqsoff_hist, NULL);
  19632. + for_each_online_cpu(cpu) {
  19633. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  19634. + per_cpu(hist_irqsoff_counting,
  19635. + cpu) = 0;
  19636. +#endif
  19637. +#ifdef CONFIG_PREEMPT_OFF_HIST
  19638. + per_cpu(hist_preemptoff_counting,
  19639. + cpu) = 0;
  19640. +#endif
  19641. +#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
  19642. + per_cpu(hist_preemptirqsoff_counting,
  19643. + cpu) = 0;
  19644. +#endif
  19645. + }
  19646. + }
  19647. + break;
  19648. +#endif
  19649. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  19650. + case WAKEUP_LATENCY:
  19651. + {
  19652. + int cpu;
  19653. +
  19654. + unregister_trace_sched_wakeup(
  19655. + probe_wakeup_latency_hist_start, NULL);
  19656. + unregister_trace_sched_wakeup_new(
  19657. + probe_wakeup_latency_hist_start, NULL);
  19658. + unregister_trace_sched_switch(
  19659. + probe_wakeup_latency_hist_stop, NULL);
  19660. + unregister_trace_sched_migrate_task(
  19661. + probe_sched_migrate_task, NULL);
  19662. +
  19663. + for_each_online_cpu(cpu) {
  19664. + per_cpu(wakeup_task, cpu) = NULL;
  19665. + per_cpu(wakeup_sharedprio, cpu) = 0;
  19666. + }
  19667. + }
  19668. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  19669. + timerandwakeup_enabled_data.enabled = 0;
  19670. +#endif
  19671. + break;
  19672. +#endif
  19673. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  19674. + case MISSED_TIMER_OFFSETS:
  19675. + unregister_trace_hrtimer_interrupt(
  19676. + probe_hrtimer_interrupt, NULL);
  19677. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  19678. + timerandwakeup_enabled_data.enabled = 0;
  19679. +#endif
  19680. + break;
  19681. +#endif
  19682. + default:
  19683. + break;
  19684. + }
  19685. + }
  19686. + ed->enabled = enable;
  19687. + return cnt;
  19688. +}
  19689. +
  19690. +static const struct file_operations latency_hist_reset_fops = {
  19691. + .open = tracing_open_generic,
  19692. + .write = latency_hist_reset,
  19693. +};
  19694. +
  19695. +static const struct file_operations enable_fops = {
  19696. + .open = tracing_open_generic,
  19697. + .read = show_enable,
  19698. + .write = do_enable,
  19699. +};
  19700. +
  19701. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  19702. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19703. +static const struct file_operations pid_fops = {
  19704. + .open = tracing_open_generic,
  19705. + .read = show_pid,
  19706. + .write = do_pid,
  19707. +};
  19708. +
  19709. +static const struct file_operations maxlatproc_fops = {
  19710. + .open = tracing_open_generic,
  19711. + .read = show_maxlatproc,
  19712. +};
  19713. +#endif
  19714. +
  19715. +#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST)
  19716. +static notrace void probe_preemptirqsoff_hist(void *v, int reason,
  19717. + int starthist)
  19718. +{
  19719. + int cpu = raw_smp_processor_id();
  19720. + int time_set = 0;
  19721. +
  19722. + if (starthist) {
  19723. + cycle_t uninitialized_var(start);
  19724. +
  19725. + if (!preempt_count() && !irqs_disabled())
  19726. + return;
  19727. +
  19728. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  19729. + if ((reason == IRQS_OFF || reason == TRACE_START) &&
  19730. + !per_cpu(hist_irqsoff_counting, cpu)) {
  19731. + per_cpu(hist_irqsoff_counting, cpu) = 1;
  19732. + start = ftrace_now(cpu);
  19733. + time_set++;
  19734. + per_cpu(hist_irqsoff_start, cpu) = start;
  19735. + }
  19736. +#endif
  19737. +
  19738. +#ifdef CONFIG_PREEMPT_OFF_HIST
  19739. + if ((reason == PREEMPT_OFF || reason == TRACE_START) &&
  19740. + !per_cpu(hist_preemptoff_counting, cpu)) {
  19741. + per_cpu(hist_preemptoff_counting, cpu) = 1;
  19742. + if (!(time_set++))
  19743. + start = ftrace_now(cpu);
  19744. + per_cpu(hist_preemptoff_start, cpu) = start;
  19745. + }
  19746. +#endif
  19747. +
  19748. +#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
  19749. + if (per_cpu(hist_irqsoff_counting, cpu) &&
  19750. + per_cpu(hist_preemptoff_counting, cpu) &&
  19751. + !per_cpu(hist_preemptirqsoff_counting, cpu)) {
  19752. + per_cpu(hist_preemptirqsoff_counting, cpu) = 1;
  19753. + if (!time_set)
  19754. + start = ftrace_now(cpu);
  19755. + per_cpu(hist_preemptirqsoff_start, cpu) = start;
  19756. + }
  19757. +#endif
  19758. + } else {
  19759. + cycle_t uninitialized_var(stop);
  19760. +
  19761. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  19762. + if ((reason == IRQS_ON || reason == TRACE_STOP) &&
  19763. + per_cpu(hist_irqsoff_counting, cpu)) {
  19764. + cycle_t start = per_cpu(hist_irqsoff_start, cpu);
  19765. +
  19766. + stop = ftrace_now(cpu);
  19767. + time_set++;
  19768. + if (start) {
  19769. + long latency = ((long) (stop - start)) /
  19770. + NSECS_PER_USECS;
  19771. +
  19772. + latency_hist(IRQSOFF_LATENCY, cpu, latency, 0,
  19773. + stop, NULL);
  19774. + }
  19775. + per_cpu(hist_irqsoff_counting, cpu) = 0;
  19776. + }
  19777. +#endif
  19778. +
  19779. +#ifdef CONFIG_PREEMPT_OFF_HIST
  19780. + if ((reason == PREEMPT_ON || reason == TRACE_STOP) &&
  19781. + per_cpu(hist_preemptoff_counting, cpu)) {
  19782. + cycle_t start = per_cpu(hist_preemptoff_start, cpu);
  19783. +
  19784. + if (!(time_set++))
  19785. + stop = ftrace_now(cpu);
  19786. + if (start) {
  19787. + long latency = ((long) (stop - start)) /
  19788. + NSECS_PER_USECS;
  19789. +
  19790. + latency_hist(PREEMPTOFF_LATENCY, cpu, latency,
  19791. + 0, stop, NULL);
  19792. + }
  19793. + per_cpu(hist_preemptoff_counting, cpu) = 0;
  19794. + }
  19795. +#endif
  19796. +
  19797. +#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
  19798. + if ((!per_cpu(hist_irqsoff_counting, cpu) ||
  19799. + !per_cpu(hist_preemptoff_counting, cpu)) &&
  19800. + per_cpu(hist_preemptirqsoff_counting, cpu)) {
  19801. + cycle_t start = per_cpu(hist_preemptirqsoff_start, cpu);
  19802. +
  19803. + if (!time_set)
  19804. + stop = ftrace_now(cpu);
  19805. + if (start) {
  19806. + long latency = ((long) (stop - start)) /
  19807. + NSECS_PER_USECS;
  19808. +
  19809. + latency_hist(PREEMPTIRQSOFF_LATENCY, cpu,
  19810. + latency, 0, stop, NULL);
  19811. + }
  19812. + per_cpu(hist_preemptirqsoff_counting, cpu) = 0;
  19813. + }
  19814. +#endif
  19815. + }
  19816. +}
  19817. +#endif
  19818. +
  19819. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  19820. +static DEFINE_RAW_SPINLOCK(wakeup_lock);
  19821. +static notrace void probe_sched_migrate_task(void *v, struct task_struct *task,
  19822. + int cpu)
  19823. +{
  19824. + int old_cpu = task_cpu(task);
  19825. +
  19826. + if (cpu != old_cpu) {
  19827. + unsigned long flags;
  19828. + struct task_struct *cpu_wakeup_task;
  19829. +
  19830. + raw_spin_lock_irqsave(&wakeup_lock, flags);
  19831. +
  19832. + cpu_wakeup_task = per_cpu(wakeup_task, old_cpu);
  19833. + if (task == cpu_wakeup_task) {
  19834. + put_task_struct(cpu_wakeup_task);
  19835. + per_cpu(wakeup_task, old_cpu) = NULL;
  19836. + cpu_wakeup_task = per_cpu(wakeup_task, cpu) = task;
  19837. + get_task_struct(cpu_wakeup_task);
  19838. + }
  19839. +
  19840. + raw_spin_unlock_irqrestore(&wakeup_lock, flags);
  19841. + }
  19842. +}
  19843. +
  19844. +static notrace void probe_wakeup_latency_hist_start(void *v,
  19845. + struct task_struct *p)
  19846. +{
  19847. + unsigned long flags;
  19848. + struct task_struct *curr = current;
  19849. + int cpu = task_cpu(p);
  19850. + struct task_struct *cpu_wakeup_task;
  19851. +
  19852. + raw_spin_lock_irqsave(&wakeup_lock, flags);
  19853. +
  19854. + cpu_wakeup_task = per_cpu(wakeup_task, cpu);
  19855. +
  19856. + if (wakeup_pid) {
  19857. + if ((cpu_wakeup_task && p->prio == cpu_wakeup_task->prio) ||
  19858. + p->prio == curr->prio)
  19859. + per_cpu(wakeup_sharedprio, cpu) = 1;
  19860. + if (likely(wakeup_pid != task_pid_nr(p)))
  19861. + goto out;
  19862. + } else {
  19863. + if (likely(!rt_task(p)) ||
  19864. + (cpu_wakeup_task && p->prio > cpu_wakeup_task->prio) ||
  19865. + p->prio > curr->prio)
  19866. + goto out;
  19867. + if ((cpu_wakeup_task && p->prio == cpu_wakeup_task->prio) ||
  19868. + p->prio == curr->prio)
  19869. + per_cpu(wakeup_sharedprio, cpu) = 1;
  19870. + }
  19871. +
  19872. + if (cpu_wakeup_task)
  19873. + put_task_struct(cpu_wakeup_task);
  19874. + cpu_wakeup_task = per_cpu(wakeup_task, cpu) = p;
  19875. + get_task_struct(cpu_wakeup_task);
  19876. + cpu_wakeup_task->preempt_timestamp_hist =
  19877. + ftrace_now(raw_smp_processor_id());
  19878. +out:
  19879. + raw_spin_unlock_irqrestore(&wakeup_lock, flags);
  19880. +}
  19881. +
  19882. +static notrace void probe_wakeup_latency_hist_stop(void *v,
  19883. + bool preempt, struct task_struct *prev, struct task_struct *next)
  19884. +{
  19885. + unsigned long flags;
  19886. + int cpu = task_cpu(next);
  19887. + long latency;
  19888. + cycle_t stop;
  19889. + struct task_struct *cpu_wakeup_task;
  19890. +
  19891. + raw_spin_lock_irqsave(&wakeup_lock, flags);
  19892. +
  19893. + cpu_wakeup_task = per_cpu(wakeup_task, cpu);
  19894. +
  19895. + if (cpu_wakeup_task == NULL)
  19896. + goto out;
  19897. +
  19898. + /* Already running? */
  19899. + if (unlikely(current == cpu_wakeup_task))
  19900. + goto out_reset;
  19901. +
  19902. + if (next != cpu_wakeup_task) {
  19903. + if (next->prio < cpu_wakeup_task->prio)
  19904. + goto out_reset;
  19905. +
  19906. + if (next->prio == cpu_wakeup_task->prio)
  19907. + per_cpu(wakeup_sharedprio, cpu) = 1;
  19908. +
  19909. + goto out;
  19910. + }
  19911. +
  19912. + if (current->prio == cpu_wakeup_task->prio)
  19913. + per_cpu(wakeup_sharedprio, cpu) = 1;
  19914. +
  19915. + /*
  19916. + * The task we are waiting for is about to be switched to.
  19917. + * Calculate latency and store it in histogram.
  19918. + */
  19919. + stop = ftrace_now(raw_smp_processor_id());
  19920. +
  19921. + latency = ((long) (stop - next->preempt_timestamp_hist)) /
  19922. + NSECS_PER_USECS;
  19923. +
  19924. + if (per_cpu(wakeup_sharedprio, cpu)) {
  19925. + latency_hist(WAKEUP_LATENCY_SHAREDPRIO, cpu, latency, 0, stop,
  19926. + next);
  19927. + per_cpu(wakeup_sharedprio, cpu) = 0;
  19928. + } else {
  19929. + latency_hist(WAKEUP_LATENCY, cpu, latency, 0, stop, next);
  19930. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  19931. + if (timerandwakeup_enabled_data.enabled) {
  19932. + latency_hist(TIMERANDWAKEUP_LATENCY, cpu,
  19933. + next->timer_offset + latency, next->timer_offset,
  19934. + stop, next);
  19935. + }
  19936. +#endif
  19937. + }
  19938. +
  19939. +out_reset:
  19940. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  19941. + next->timer_offset = 0;
  19942. +#endif
  19943. + put_task_struct(cpu_wakeup_task);
  19944. + per_cpu(wakeup_task, cpu) = NULL;
  19945. +out:
  19946. + raw_spin_unlock_irqrestore(&wakeup_lock, flags);
  19947. +}
  19948. +#endif
  19949. +
  19950. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  19951. +static notrace void probe_hrtimer_interrupt(void *v, int cpu,
  19952. + long long latency_ns, struct task_struct *curr,
  19953. + struct task_struct *task)
  19954. +{
  19955. + if (latency_ns <= 0 && task != NULL && rt_task(task) &&
  19956. + (task->prio < curr->prio ||
  19957. + (task->prio == curr->prio &&
  19958. + !cpumask_test_cpu(cpu, &task->cpus_allowed)))) {
  19959. + long latency;
  19960. + cycle_t now;
  19961. +
  19962. + if (missed_timer_offsets_pid) {
  19963. + if (likely(missed_timer_offsets_pid !=
  19964. + task_pid_nr(task)))
  19965. + return;
  19966. + }
  19967. +
  19968. + now = ftrace_now(cpu);
  19969. + latency = (long) div_s64(-latency_ns, NSECS_PER_USECS);
  19970. + latency_hist(MISSED_TIMER_OFFSETS, cpu, latency, latency, now,
  19971. + task);
  19972. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  19973. + task->timer_offset = latency;
  19974. +#endif
  19975. + }
  19976. +}
  19977. +#endif
  19978. +
  19979. +static __init int latency_hist_init(void)
  19980. +{
  19981. + struct dentry *latency_hist_root = NULL;
  19982. + struct dentry *dentry;
  19983. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  19984. + struct dentry *dentry_sharedprio;
  19985. +#endif
  19986. + struct dentry *entry;
  19987. + struct dentry *enable_root;
  19988. + int i = 0;
  19989. + struct hist_data *my_hist;
  19990. + char name[64];
  19991. + char *cpufmt = "CPU%d";
  19992. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
  19993. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  19994. + char *cpufmt_maxlatproc = "max_latency-CPU%d";
  19995. + struct maxlatproc_data *mp = NULL;
  19996. +#endif
  19997. +
  19998. + dentry = tracing_init_dentry();
  19999. + latency_hist_root = debugfs_create_dir(latency_hist_dir_root, dentry);
  20000. + enable_root = debugfs_create_dir("enable", latency_hist_root);
  20001. +
  20002. +#ifdef CONFIG_INTERRUPT_OFF_HIST
  20003. + dentry = debugfs_create_dir(irqsoff_hist_dir, latency_hist_root);
  20004. + for_each_possible_cpu(i) {
  20005. + sprintf(name, cpufmt, i);
  20006. + entry = debugfs_create_file(name, 0444, dentry,
  20007. + &per_cpu(irqsoff_hist, i), &latency_hist_fops);
  20008. + my_hist = &per_cpu(irqsoff_hist, i);
  20009. + atomic_set(&my_hist->hist_mode, 1);
  20010. + my_hist->min_lat = LONG_MAX;
  20011. + }
  20012. + entry = debugfs_create_file("reset", 0644, dentry,
  20013. + (void *)IRQSOFF_LATENCY, &latency_hist_reset_fops);
  20014. +#endif
  20015. +
  20016. +#ifdef CONFIG_PREEMPT_OFF_HIST
  20017. + dentry = debugfs_create_dir(preemptoff_hist_dir,
  20018. + latency_hist_root);
  20019. + for_each_possible_cpu(i) {
  20020. + sprintf(name, cpufmt, i);
  20021. + entry = debugfs_create_file(name, 0444, dentry,
  20022. + &per_cpu(preemptoff_hist, i), &latency_hist_fops);
  20023. + my_hist = &per_cpu(preemptoff_hist, i);
  20024. + atomic_set(&my_hist->hist_mode, 1);
  20025. + my_hist->min_lat = LONG_MAX;
  20026. + }
  20027. + entry = debugfs_create_file("reset", 0644, dentry,
  20028. + (void *)PREEMPTOFF_LATENCY, &latency_hist_reset_fops);
  20029. +#endif
  20030. +
  20031. +#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
  20032. + dentry = debugfs_create_dir(preemptirqsoff_hist_dir,
  20033. + latency_hist_root);
  20034. + for_each_possible_cpu(i) {
  20035. + sprintf(name, cpufmt, i);
  20036. + entry = debugfs_create_file(name, 0444, dentry,
  20037. + &per_cpu(preemptirqsoff_hist, i), &latency_hist_fops);
  20038. + my_hist = &per_cpu(preemptirqsoff_hist, i);
  20039. + atomic_set(&my_hist->hist_mode, 1);
  20040. + my_hist->min_lat = LONG_MAX;
  20041. + }
  20042. + entry = debugfs_create_file("reset", 0644, dentry,
  20043. + (void *)PREEMPTIRQSOFF_LATENCY, &latency_hist_reset_fops);
  20044. +#endif
  20045. +
  20046. +#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST)
  20047. + entry = debugfs_create_file("preemptirqsoff", 0644,
  20048. + enable_root, (void *)&preemptirqsoff_enabled_data,
  20049. + &enable_fops);
  20050. +#endif
  20051. +
  20052. +#ifdef CONFIG_WAKEUP_LATENCY_HIST
  20053. + dentry = debugfs_create_dir(wakeup_latency_hist_dir,
  20054. + latency_hist_root);
  20055. + dentry_sharedprio = debugfs_create_dir(
  20056. + wakeup_latency_hist_dir_sharedprio, dentry);
  20057. + for_each_possible_cpu(i) {
  20058. + sprintf(name, cpufmt, i);
  20059. +
  20060. + entry = debugfs_create_file(name, 0444, dentry,
  20061. + &per_cpu(wakeup_latency_hist, i),
  20062. + &latency_hist_fops);
  20063. + my_hist = &per_cpu(wakeup_latency_hist, i);
  20064. + atomic_set(&my_hist->hist_mode, 1);
  20065. + my_hist->min_lat = LONG_MAX;
  20066. +
  20067. + entry = debugfs_create_file(name, 0444, dentry_sharedprio,
  20068. + &per_cpu(wakeup_latency_hist_sharedprio, i),
  20069. + &latency_hist_fops);
  20070. + my_hist = &per_cpu(wakeup_latency_hist_sharedprio, i);
  20071. + atomic_set(&my_hist->hist_mode, 1);
  20072. + my_hist->min_lat = LONG_MAX;
  20073. +
  20074. + sprintf(name, cpufmt_maxlatproc, i);
  20075. +
  20076. + mp = &per_cpu(wakeup_maxlatproc, i);
  20077. + entry = debugfs_create_file(name, 0444, dentry, mp,
  20078. + &maxlatproc_fops);
  20079. + clear_maxlatprocdata(mp);
  20080. +
  20081. + mp = &per_cpu(wakeup_maxlatproc_sharedprio, i);
  20082. + entry = debugfs_create_file(name, 0444, dentry_sharedprio, mp,
  20083. + &maxlatproc_fops);
  20084. + clear_maxlatprocdata(mp);
  20085. + }
  20086. + entry = debugfs_create_file("pid", 0644, dentry,
  20087. + (void *)&wakeup_pid, &pid_fops);
  20088. + entry = debugfs_create_file("reset", 0644, dentry,
  20089. + (void *)WAKEUP_LATENCY, &latency_hist_reset_fops);
  20090. + entry = debugfs_create_file("reset", 0644, dentry_sharedprio,
  20091. + (void *)WAKEUP_LATENCY_SHAREDPRIO, &latency_hist_reset_fops);
  20092. + entry = debugfs_create_file("wakeup", 0644,
  20093. + enable_root, (void *)&wakeup_latency_enabled_data,
  20094. + &enable_fops);
  20095. +#endif
  20096. +
  20097. +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
  20098. + dentry = debugfs_create_dir(missed_timer_offsets_dir,
  20099. + latency_hist_root);
  20100. + for_each_possible_cpu(i) {
  20101. + sprintf(name, cpufmt, i);
  20102. + entry = debugfs_create_file(name, 0444, dentry,
  20103. + &per_cpu(missed_timer_offsets, i), &latency_hist_fops);
  20104. + my_hist = &per_cpu(missed_timer_offsets, i);
  20105. + atomic_set(&my_hist->hist_mode, 1);
  20106. + my_hist->min_lat = LONG_MAX;
  20107. +
  20108. + sprintf(name, cpufmt_maxlatproc, i);
  20109. + mp = &per_cpu(missed_timer_offsets_maxlatproc, i);
  20110. + entry = debugfs_create_file(name, 0444, dentry, mp,
  20111. + &maxlatproc_fops);
  20112. + clear_maxlatprocdata(mp);
  20113. + }
  20114. + entry = debugfs_create_file("pid", 0644, dentry,
  20115. + (void *)&missed_timer_offsets_pid, &pid_fops);
  20116. + entry = debugfs_create_file("reset", 0644, dentry,
  20117. + (void *)MISSED_TIMER_OFFSETS, &latency_hist_reset_fops);
  20118. + entry = debugfs_create_file("missed_timer_offsets", 0644,
  20119. + enable_root, (void *)&missed_timer_offsets_enabled_data,
  20120. + &enable_fops);
  20121. +#endif
  20122. +
  20123. +#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \
  20124. + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
  20125. + dentry = debugfs_create_dir(timerandwakeup_latency_hist_dir,
  20126. + latency_hist_root);
  20127. + for_each_possible_cpu(i) {
  20128. + sprintf(name, cpufmt, i);
  20129. + entry = debugfs_create_file(name, 0444, dentry,
  20130. + &per_cpu(timerandwakeup_latency_hist, i),
  20131. + &latency_hist_fops);
  20132. + my_hist = &per_cpu(timerandwakeup_latency_hist, i);
  20133. + atomic_set(&my_hist->hist_mode, 1);
  20134. + my_hist->min_lat = LONG_MAX;
  20135. +
  20136. + sprintf(name, cpufmt_maxlatproc, i);
  20137. + mp = &per_cpu(timerandwakeup_maxlatproc, i);
  20138. + entry = debugfs_create_file(name, 0444, dentry, mp,
  20139. + &maxlatproc_fops);
  20140. + clear_maxlatprocdata(mp);
  20141. + }
  20142. + entry = debugfs_create_file("reset", 0644, dentry,
  20143. + (void *)TIMERANDWAKEUP_LATENCY, &latency_hist_reset_fops);
  20144. + entry = debugfs_create_file("timerandwakeup", 0644,
  20145. + enable_root, (void *)&timerandwakeup_enabled_data,
  20146. + &enable_fops);
  20147. +#endif
  20148. + return 0;
  20149. +}
  20150. +
  20151. +device_initcall(latency_hist_init);
  20152. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/trace.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/trace.c
  20153. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/trace.c 2017-04-16 10:38:30.000000000 +0200
  20154. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/trace.c 2017-04-18 17:54:26.000000000 +0200
  20155. @@ -1897,6 +1897,7 @@
  20156. struct task_struct *tsk = current;
  20157. entry->preempt_count = pc & 0xff;
  20158. + entry->preempt_lazy_count = preempt_lazy_count();
  20159. entry->pid = (tsk) ? tsk->pid : 0;
  20160. entry->flags =
  20161. #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
  20162. @@ -1907,8 +1908,11 @@
  20163. ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
  20164. ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
  20165. ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
  20166. - (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
  20167. + (tif_need_resched_now() ? TRACE_FLAG_NEED_RESCHED : 0) |
  20168. + (need_resched_lazy() ? TRACE_FLAG_NEED_RESCHED_LAZY : 0) |
  20169. (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
  20170. +
  20171. + entry->migrate_disable = (tsk) ? __migrate_disabled(tsk) & 0xFF : 0;
  20172. }
  20173. EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
  20174. @@ -2892,14 +2896,17 @@
  20175. static void print_lat_help_header(struct seq_file *m)
  20176. {
  20177. - seq_puts(m, "# _------=> CPU# \n"
  20178. - "# / _-----=> irqs-off \n"
  20179. - "# | / _----=> need-resched \n"
  20180. - "# || / _---=> hardirq/softirq \n"
  20181. - "# ||| / _--=> preempt-depth \n"
  20182. - "# |||| / delay \n"
  20183. - "# cmd pid ||||| time | caller \n"
  20184. - "# \\ / ||||| \\ | / \n");
  20185. + seq_puts(m, "# _--------=> CPU# \n"
  20186. + "# / _-------=> irqs-off \n"
  20187. + "# | / _------=> need-resched \n"
  20188. + "# || / _-----=> need-resched_lazy \n"
  20189. + "# ||| / _----=> hardirq/softirq \n"
  20190. + "# |||| / _---=> preempt-depth \n"
  20191. + "# ||||| / _--=> preempt-lazy-depth\n"
  20192. + "# |||||| / _-=> migrate-disable \n"
  20193. + "# ||||||| / delay \n"
  20194. + "# cmd pid |||||||| time | caller \n"
  20195. + "# \\ / |||||||| \\ | / \n");
  20196. }
  20197. static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
  20198. @@ -2925,11 +2932,14 @@
  20199. print_event_info(buf, m);
  20200. seq_puts(m, "# _-----=> irqs-off\n"
  20201. "# / _----=> need-resched\n"
  20202. - "# | / _---=> hardirq/softirq\n"
  20203. - "# || / _--=> preempt-depth\n"
  20204. - "# ||| / delay\n"
  20205. - "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
  20206. - "# | | | |||| | |\n");
  20207. + "# |/ _-----=> need-resched_lazy\n"
  20208. + "# || / _---=> hardirq/softirq\n"
  20209. + "# ||| / _--=> preempt-depth\n"
  20210. + "# |||| / _-=> preempt-lazy-depth\n"
  20211. + "# ||||| / _-=> migrate-disable \n"
  20212. + "# |||||| / delay\n"
  20213. + "# TASK-PID CPU# ||||||| TIMESTAMP FUNCTION\n"
  20214. + "# | | | ||||||| | |\n");
  20215. }
  20216. void
  20217. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/trace.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/trace.h
  20218. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/trace.h 2017-04-16 10:38:30.000000000 +0200
  20219. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/trace.h 2017-04-18 17:54:26.000000000 +0200
  20220. @@ -124,6 +124,7 @@
  20221. * NEED_RESCHED - reschedule is requested
  20222. * HARDIRQ - inside an interrupt handler
  20223. * SOFTIRQ - inside a softirq handler
  20224. + * NEED_RESCHED_LAZY - lazy reschedule is requested
  20225. */
  20226. enum trace_flag_type {
  20227. TRACE_FLAG_IRQS_OFF = 0x01,
  20228. @@ -133,6 +134,7 @@
  20229. TRACE_FLAG_SOFTIRQ = 0x10,
  20230. TRACE_FLAG_PREEMPT_RESCHED = 0x20,
  20231. TRACE_FLAG_NMI = 0x40,
  20232. + TRACE_FLAG_NEED_RESCHED_LAZY = 0x80,
  20233. };
  20234. #define TRACE_BUF_SIZE 1024
  20235. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/trace_events.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/trace_events.c
  20236. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/trace_events.c 2017-04-16 10:38:30.000000000 +0200
  20237. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/trace_events.c 2017-04-18 17:54:26.000000000 +0200
  20238. @@ -187,6 +187,8 @@
  20239. __common_field(unsigned char, flags);
  20240. __common_field(unsigned char, preempt_count);
  20241. __common_field(int, pid);
  20242. + __common_field(unsigned short, migrate_disable);
  20243. + __common_field(unsigned short, padding);
  20244. return ret;
  20245. }
  20246. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/trace_irqsoff.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/trace_irqsoff.c
  20247. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/trace_irqsoff.c 2017-04-16 10:38:30.000000000 +0200
  20248. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/trace_irqsoff.c 2017-04-18 17:54:26.000000000 +0200
  20249. @@ -13,6 +13,7 @@
  20250. #include <linux/uaccess.h>
  20251. #include <linux/module.h>
  20252. #include <linux/ftrace.h>
  20253. +#include <trace/events/hist.h>
  20254. #include "trace.h"
  20255. @@ -424,11 +425,13 @@
  20256. {
  20257. if (preempt_trace() || irq_trace())
  20258. start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
  20259. + trace_preemptirqsoff_hist_rcuidle(TRACE_START, 1);
  20260. }
  20261. EXPORT_SYMBOL_GPL(start_critical_timings);
  20262. void stop_critical_timings(void)
  20263. {
  20264. + trace_preemptirqsoff_hist_rcuidle(TRACE_STOP, 0);
  20265. if (preempt_trace() || irq_trace())
  20266. stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
  20267. }
  20268. @@ -438,6 +441,7 @@
  20269. #ifdef CONFIG_PROVE_LOCKING
  20270. void time_hardirqs_on(unsigned long a0, unsigned long a1)
  20271. {
  20272. + trace_preemptirqsoff_hist_rcuidle(IRQS_ON, 0);
  20273. if (!preempt_trace() && irq_trace())
  20274. stop_critical_timing(a0, a1);
  20275. }
  20276. @@ -446,6 +450,7 @@
  20277. {
  20278. if (!preempt_trace() && irq_trace())
  20279. start_critical_timing(a0, a1);
  20280. + trace_preemptirqsoff_hist_rcuidle(IRQS_OFF, 1);
  20281. }
  20282. #else /* !CONFIG_PROVE_LOCKING */
  20283. @@ -471,6 +476,7 @@
  20284. */
  20285. void trace_hardirqs_on(void)
  20286. {
  20287. + trace_preemptirqsoff_hist(IRQS_ON, 0);
  20288. if (!preempt_trace() && irq_trace())
  20289. stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
  20290. }
  20291. @@ -480,11 +486,13 @@
  20292. {
  20293. if (!preempt_trace() && irq_trace())
  20294. start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
  20295. + trace_preemptirqsoff_hist(IRQS_OFF, 1);
  20296. }
  20297. EXPORT_SYMBOL(trace_hardirqs_off);
  20298. __visible void trace_hardirqs_on_caller(unsigned long caller_addr)
  20299. {
  20300. + trace_preemptirqsoff_hist(IRQS_ON, 0);
  20301. if (!preempt_trace() && irq_trace())
  20302. stop_critical_timing(CALLER_ADDR0, caller_addr);
  20303. }
  20304. @@ -494,6 +502,7 @@
  20305. {
  20306. if (!preempt_trace() && irq_trace())
  20307. start_critical_timing(CALLER_ADDR0, caller_addr);
  20308. + trace_preemptirqsoff_hist(IRQS_OFF, 1);
  20309. }
  20310. EXPORT_SYMBOL(trace_hardirqs_off_caller);
  20311. @@ -503,12 +512,14 @@
  20312. #ifdef CONFIG_PREEMPT_TRACER
  20313. void trace_preempt_on(unsigned long a0, unsigned long a1)
  20314. {
  20315. + trace_preemptirqsoff_hist(PREEMPT_ON, 0);
  20316. if (preempt_trace() && !irq_trace())
  20317. stop_critical_timing(a0, a1);
  20318. }
  20319. void trace_preempt_off(unsigned long a0, unsigned long a1)
  20320. {
  20321. + trace_preemptirqsoff_hist(PREEMPT_ON, 1);
  20322. if (preempt_trace() && !irq_trace())
  20323. start_critical_timing(a0, a1);
  20324. }
  20325. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/trace_output.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/trace_output.c
  20326. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/trace/trace_output.c 2017-04-16 10:38:30.000000000 +0200
  20327. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/trace/trace_output.c 2017-04-18 17:54:26.000000000 +0200
  20328. @@ -386,6 +386,7 @@
  20329. {
  20330. char hardsoft_irq;
  20331. char need_resched;
  20332. + char need_resched_lazy;
  20333. char irqs_off;
  20334. int hardirq;
  20335. int softirq;
  20336. @@ -416,6 +417,9 @@
  20337. break;
  20338. }
  20339. + need_resched_lazy =
  20340. + (entry->flags & TRACE_FLAG_NEED_RESCHED_LAZY) ? 'L' : '.';
  20341. +
  20342. hardsoft_irq =
  20343. (nmi && hardirq) ? 'Z' :
  20344. nmi ? 'z' :
  20345. @@ -424,14 +428,25 @@
  20346. softirq ? 's' :
  20347. '.' ;
  20348. - trace_seq_printf(s, "%c%c%c",
  20349. - irqs_off, need_resched, hardsoft_irq);
  20350. + trace_seq_printf(s, "%c%c%c%c",
  20351. + irqs_off, need_resched, need_resched_lazy,
  20352. + hardsoft_irq);
  20353. if (entry->preempt_count)
  20354. trace_seq_printf(s, "%x", entry->preempt_count);
  20355. else
  20356. trace_seq_putc(s, '.');
  20357. + if (entry->preempt_lazy_count)
  20358. + trace_seq_printf(s, "%x", entry->preempt_lazy_count);
  20359. + else
  20360. + trace_seq_putc(s, '.');
  20361. +
  20362. + if (entry->migrate_disable)
  20363. + trace_seq_printf(s, "%x", entry->migrate_disable);
  20364. + else
  20365. + trace_seq_putc(s, '.');
  20366. +
  20367. return !trace_seq_has_overflowed(s);
  20368. }
  20369. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/user.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/user.c
  20370. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/user.c 2017-04-16 10:38:30.000000000 +0200
  20371. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/user.c 2017-04-18 17:54:26.000000000 +0200
  20372. @@ -161,11 +161,11 @@
  20373. if (!up)
  20374. return;
  20375. - local_irq_save(flags);
  20376. + local_irq_save_nort(flags);
  20377. if (atomic_dec_and_lock(&up->__count, &uidhash_lock))
  20378. free_user(up, flags);
  20379. else
  20380. - local_irq_restore(flags);
  20381. + local_irq_restore_nort(flags);
  20382. }
  20383. struct user_struct *alloc_uid(kuid_t uid)
  20384. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/watchdog.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/watchdog.c
  20385. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/watchdog.c 2017-04-16 10:38:30.000000000 +0200
  20386. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/watchdog.c 2017-04-18 17:54:26.000000000 +0200
  20387. @@ -315,6 +315,8 @@
  20388. #ifdef CONFIG_HARDLOCKUP_DETECTOR
  20389. +static DEFINE_RAW_SPINLOCK(watchdog_output_lock);
  20390. +
  20391. static struct perf_event_attr wd_hw_attr = {
  20392. .type = PERF_TYPE_HARDWARE,
  20393. .config = PERF_COUNT_HW_CPU_CYCLES,
  20394. @@ -348,6 +350,13 @@
  20395. /* only print hardlockups once */
  20396. if (__this_cpu_read(hard_watchdog_warn) == true)
  20397. return;
  20398. + /*
  20399. + * If early-printk is enabled then make sure we do not
  20400. + * lock up in printk() and kill console logging:
  20401. + */
  20402. + printk_kill();
  20403. +
  20404. + raw_spin_lock(&watchdog_output_lock);
  20405. pr_emerg("Watchdog detected hard LOCKUP on cpu %d", this_cpu);
  20406. print_modules();
  20407. @@ -365,6 +374,7 @@
  20408. !test_and_set_bit(0, &hardlockup_allcpu_dumped))
  20409. trigger_allbutself_cpu_backtrace();
  20410. + raw_spin_unlock(&watchdog_output_lock);
  20411. if (hardlockup_panic)
  20412. nmi_panic(regs, "Hard LOCKUP");
  20413. @@ -512,6 +522,7 @@
  20414. /* kick off the timer for the hardlockup detector */
  20415. hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  20416. hrtimer->function = watchdog_timer_fn;
  20417. + hrtimer->irqsafe = 1;
  20418. /* Enable the perf event */
  20419. watchdog_nmi_enable(cpu);
  20420. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/workqueue.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/workqueue.c
  20421. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/workqueue.c 2017-04-16 10:38:30.000000000 +0200
  20422. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/workqueue.c 2017-04-18 17:54:26.000000000 +0200
  20423. @@ -48,6 +48,8 @@
  20424. #include <linux/nodemask.h>
  20425. #include <linux/moduleparam.h>
  20426. #include <linux/uaccess.h>
  20427. +#include <linux/locallock.h>
  20428. +#include <linux/delay.h>
  20429. #include "workqueue_internal.h"
  20430. @@ -121,11 +123,16 @@
  20431. * cpu or grabbing pool->lock is enough for read access. If
  20432. * POOL_DISASSOCIATED is set, it's identical to L.
  20433. *
  20434. + * On RT we need the extra protection via rt_lock_idle_list() for
  20435. + * the list manipulations against read access from
  20436. + * wq_worker_sleeping(). All other places are nicely serialized via
  20437. + * pool->lock.
  20438. + *
  20439. * A: pool->attach_mutex protected.
  20440. *
  20441. * PL: wq_pool_mutex protected.
  20442. *
  20443. - * PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads.
  20444. + * PR: wq_pool_mutex protected for writes. RCU protected for reads.
  20445. *
  20446. * PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
  20447. *
  20448. @@ -134,7 +141,7 @@
  20449. *
  20450. * WQ: wq->mutex protected.
  20451. *
  20452. - * WR: wq->mutex protected for writes. Sched-RCU protected for reads.
  20453. + * WR: wq->mutex protected for writes. RCU protected for reads.
  20454. *
  20455. * MD: wq_mayday_lock protected.
  20456. */
  20457. @@ -185,7 +192,7 @@
  20458. atomic_t nr_running ____cacheline_aligned_in_smp;
  20459. /*
  20460. - * Destruction of pool is sched-RCU protected to allow dereferences
  20461. + * Destruction of pool is RCU protected to allow dereferences
  20462. * from get_work_pool().
  20463. */
  20464. struct rcu_head rcu;
  20465. @@ -214,7 +221,7 @@
  20466. /*
  20467. * Release of unbound pwq is punted to system_wq. See put_pwq()
  20468. * and pwq_unbound_release_workfn() for details. pool_workqueue
  20469. - * itself is also sched-RCU protected so that the first pwq can be
  20470. + * itself is also RCU protected so that the first pwq can be
  20471. * determined without grabbing wq->mutex.
  20472. */
  20473. struct work_struct unbound_release_work;
  20474. @@ -348,6 +355,8 @@
  20475. struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
  20476. EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
  20477. +static DEFINE_LOCAL_IRQ_LOCK(pendingb_lock);
  20478. +
  20479. static int worker_thread(void *__worker);
  20480. static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
  20481. @@ -355,20 +364,20 @@
  20482. #include <trace/events/workqueue.h>
  20483. #define assert_rcu_or_pool_mutex() \
  20484. - RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
  20485. + RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
  20486. !lockdep_is_held(&wq_pool_mutex), \
  20487. - "sched RCU or wq_pool_mutex should be held")
  20488. + "RCU or wq_pool_mutex should be held")
  20489. #define assert_rcu_or_wq_mutex(wq) \
  20490. - RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
  20491. + RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
  20492. !lockdep_is_held(&wq->mutex), \
  20493. - "sched RCU or wq->mutex should be held")
  20494. + "RCU or wq->mutex should be held")
  20495. #define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
  20496. - RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
  20497. + RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
  20498. !lockdep_is_held(&wq->mutex) && \
  20499. !lockdep_is_held(&wq_pool_mutex), \
  20500. - "sched RCU, wq->mutex or wq_pool_mutex should be held")
  20501. + "RCU, wq->mutex or wq_pool_mutex should be held")
  20502. #define for_each_cpu_worker_pool(pool, cpu) \
  20503. for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
  20504. @@ -380,7 +389,7 @@
  20505. * @pool: iteration cursor
  20506. * @pi: integer used for iteration
  20507. *
  20508. - * This must be called either with wq_pool_mutex held or sched RCU read
  20509. + * This must be called either with wq_pool_mutex held or RCU read
  20510. * locked. If the pool needs to be used beyond the locking in effect, the
  20511. * caller is responsible for guaranteeing that the pool stays online.
  20512. *
  20513. @@ -412,7 +421,7 @@
  20514. * @pwq: iteration cursor
  20515. * @wq: the target workqueue
  20516. *
  20517. - * This must be called either with wq->mutex held or sched RCU read locked.
  20518. + * This must be called either with wq->mutex held or RCU read locked.
  20519. * If the pwq needs to be used beyond the locking in effect, the caller is
  20520. * responsible for guaranteeing that the pwq stays online.
  20521. *
  20522. @@ -424,6 +433,31 @@
  20523. if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
  20524. else
  20525. +#ifdef CONFIG_PREEMPT_RT_BASE
  20526. +static inline void rt_lock_idle_list(struct worker_pool *pool)
  20527. +{
  20528. + preempt_disable();
  20529. +}
  20530. +static inline void rt_unlock_idle_list(struct worker_pool *pool)
  20531. +{
  20532. + preempt_enable();
  20533. +}
  20534. +static inline void sched_lock_idle_list(struct worker_pool *pool) { }
  20535. +static inline void sched_unlock_idle_list(struct worker_pool *pool) { }
  20536. +#else
  20537. +static inline void rt_lock_idle_list(struct worker_pool *pool) { }
  20538. +static inline void rt_unlock_idle_list(struct worker_pool *pool) { }
  20539. +static inline void sched_lock_idle_list(struct worker_pool *pool)
  20540. +{
  20541. + spin_lock_irq(&pool->lock);
  20542. +}
  20543. +static inline void sched_unlock_idle_list(struct worker_pool *pool)
  20544. +{
  20545. + spin_unlock_irq(&pool->lock);
  20546. +}
  20547. +#endif
  20548. +
  20549. +
  20550. #ifdef CONFIG_DEBUG_OBJECTS_WORK
  20551. static struct debug_obj_descr work_debug_descr;
  20552. @@ -548,7 +582,7 @@
  20553. * @wq: the target workqueue
  20554. * @node: the node ID
  20555. *
  20556. - * This must be called with any of wq_pool_mutex, wq->mutex or sched RCU
  20557. + * This must be called with any of wq_pool_mutex, wq->mutex or RCU
  20558. * read locked.
  20559. * If the pwq needs to be used beyond the locking in effect, the caller is
  20560. * responsible for guaranteeing that the pwq stays online.
  20561. @@ -692,8 +726,8 @@
  20562. * @work: the work item of interest
  20563. *
  20564. * Pools are created and destroyed under wq_pool_mutex, and allows read
  20565. - * access under sched-RCU read lock. As such, this function should be
  20566. - * called under wq_pool_mutex or with preemption disabled.
  20567. + * access under RCU read lock. As such, this function should be
  20568. + * called under wq_pool_mutex or inside of a rcu_read_lock() region.
  20569. *
  20570. * All fields of the returned pool are accessible as long as the above
  20571. * mentioned locking is in effect. If the returned pool needs to be used
  20572. @@ -830,50 +864,45 @@
  20573. */
  20574. static void wake_up_worker(struct worker_pool *pool)
  20575. {
  20576. - struct worker *worker = first_idle_worker(pool);
  20577. + struct worker *worker;
  20578. +
  20579. + rt_lock_idle_list(pool);
  20580. +
  20581. + worker = first_idle_worker(pool);
  20582. if (likely(worker))
  20583. wake_up_process(worker->task);
  20584. +
  20585. + rt_unlock_idle_list(pool);
  20586. }
  20587. /**
  20588. - * wq_worker_waking_up - a worker is waking up
  20589. + * wq_worker_running - a worker is running again
  20590. * @task: task waking up
  20591. - * @cpu: CPU @task is waking up to
  20592. *
  20593. - * This function is called during try_to_wake_up() when a worker is
  20594. - * being awoken.
  20595. - *
  20596. - * CONTEXT:
  20597. - * spin_lock_irq(rq->lock)
  20598. + * This function is called when a worker returns from schedule()
  20599. */
  20600. -void wq_worker_waking_up(struct task_struct *task, int cpu)
  20601. +void wq_worker_running(struct task_struct *task)
  20602. {
  20603. struct worker *worker = kthread_data(task);
  20604. - if (!(worker->flags & WORKER_NOT_RUNNING)) {
  20605. - WARN_ON_ONCE(worker->pool->cpu != cpu);
  20606. + if (!worker->sleeping)
  20607. + return;
  20608. + if (!(worker->flags & WORKER_NOT_RUNNING))
  20609. atomic_inc(&worker->pool->nr_running);
  20610. - }
  20611. + worker->sleeping = 0;
  20612. }
  20613. /**
  20614. * wq_worker_sleeping - a worker is going to sleep
  20615. * @task: task going to sleep
  20616. *
  20617. - * This function is called during schedule() when a busy worker is
  20618. - * going to sleep. Worker on the same cpu can be woken up by
  20619. - * returning pointer to its task.
  20620. - *
  20621. - * CONTEXT:
  20622. - * spin_lock_irq(rq->lock)
  20623. - *
  20624. - * Return:
  20625. - * Worker task on @cpu to wake up, %NULL if none.
  20626. + * This function is called from schedule() when a busy worker is
  20627. + * going to sleep.
  20628. */
  20629. -struct task_struct *wq_worker_sleeping(struct task_struct *task)
  20630. +void wq_worker_sleeping(struct task_struct *task)
  20631. {
  20632. - struct worker *worker = kthread_data(task), *to_wakeup = NULL;
  20633. + struct worker *worker = kthread_data(task);
  20634. struct worker_pool *pool;
  20635. /*
  20636. @@ -882,29 +911,26 @@
  20637. * checking NOT_RUNNING.
  20638. */
  20639. if (worker->flags & WORKER_NOT_RUNNING)
  20640. - return NULL;
  20641. + return;
  20642. pool = worker->pool;
  20643. - /* this can only happen on the local cpu */
  20644. - if (WARN_ON_ONCE(pool->cpu != raw_smp_processor_id()))
  20645. - return NULL;
  20646. + if (WARN_ON_ONCE(worker->sleeping))
  20647. + return;
  20648. +
  20649. + worker->sleeping = 1;
  20650. /*
  20651. * The counterpart of the following dec_and_test, implied mb,
  20652. * worklist not empty test sequence is in insert_work().
  20653. * Please read comment there.
  20654. - *
  20655. - * NOT_RUNNING is clear. This means that we're bound to and
  20656. - * running on the local cpu w/ rq lock held and preemption
  20657. - * disabled, which in turn means that none else could be
  20658. - * manipulating idle_list, so dereferencing idle_list without pool
  20659. - * lock is safe.
  20660. */
  20661. if (atomic_dec_and_test(&pool->nr_running) &&
  20662. - !list_empty(&pool->worklist))
  20663. - to_wakeup = first_idle_worker(pool);
  20664. - return to_wakeup ? to_wakeup->task : NULL;
  20665. + !list_empty(&pool->worklist)) {
  20666. + sched_lock_idle_list(pool);
  20667. + wake_up_worker(pool);
  20668. + sched_unlock_idle_list(pool);
  20669. + }
  20670. }
  20671. /**
  20672. @@ -1098,12 +1124,14 @@
  20673. {
  20674. if (pwq) {
  20675. /*
  20676. - * As both pwqs and pools are sched-RCU protected, the
  20677. + * As both pwqs and pools are RCU protected, the
  20678. * following lock operations are safe.
  20679. */
  20680. - spin_lock_irq(&pwq->pool->lock);
  20681. + rcu_read_lock();
  20682. + local_spin_lock_irq(pendingb_lock, &pwq->pool->lock);
  20683. put_pwq(pwq);
  20684. - spin_unlock_irq(&pwq->pool->lock);
  20685. + local_spin_unlock_irq(pendingb_lock, &pwq->pool->lock);
  20686. + rcu_read_unlock();
  20687. }
  20688. }
  20689. @@ -1207,7 +1235,7 @@
  20690. struct worker_pool *pool;
  20691. struct pool_workqueue *pwq;
  20692. - local_irq_save(*flags);
  20693. + local_lock_irqsave(pendingb_lock, *flags);
  20694. /* try to steal the timer if it exists */
  20695. if (is_dwork) {
  20696. @@ -1226,6 +1254,7 @@
  20697. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
  20698. return 0;
  20699. + rcu_read_lock();
  20700. /*
  20701. * The queueing is in progress, or it is already queued. Try to
  20702. * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
  20703. @@ -1264,14 +1293,16 @@
  20704. set_work_pool_and_keep_pending(work, pool->id);
  20705. spin_unlock(&pool->lock);
  20706. + rcu_read_unlock();
  20707. return 1;
  20708. }
  20709. spin_unlock(&pool->lock);
  20710. fail:
  20711. - local_irq_restore(*flags);
  20712. + rcu_read_unlock();
  20713. + local_unlock_irqrestore(pendingb_lock, *flags);
  20714. if (work_is_canceling(work))
  20715. return -ENOENT;
  20716. - cpu_relax();
  20717. + cpu_chill();
  20718. return -EAGAIN;
  20719. }
  20720. @@ -1373,7 +1404,7 @@
  20721. * queued or lose PENDING. Grabbing PENDING and queueing should
  20722. * happen with IRQ disabled.
  20723. */
  20724. - WARN_ON_ONCE(!irqs_disabled());
  20725. + WARN_ON_ONCE_NONRT(!irqs_disabled());
  20726. debug_work_activate(work);
  20727. @@ -1381,6 +1412,7 @@
  20728. if (unlikely(wq->flags & __WQ_DRAINING) &&
  20729. WARN_ON_ONCE(!is_chained_work(wq)))
  20730. return;
  20731. + rcu_read_lock();
  20732. retry:
  20733. if (req_cpu == WORK_CPU_UNBOUND)
  20734. cpu = wq_select_unbound_cpu(raw_smp_processor_id());
  20735. @@ -1437,10 +1469,8 @@
  20736. /* pwq determined, queue */
  20737. trace_workqueue_queue_work(req_cpu, pwq, work);
  20738. - if (WARN_ON(!list_empty(&work->entry))) {
  20739. - spin_unlock(&pwq->pool->lock);
  20740. - return;
  20741. - }
  20742. + if (WARN_ON(!list_empty(&work->entry)))
  20743. + goto out;
  20744. pwq->nr_in_flight[pwq->work_color]++;
  20745. work_flags = work_color_to_flags(pwq->work_color);
  20746. @@ -1458,7 +1488,9 @@
  20747. insert_work(pwq, work, worklist, work_flags);
  20748. +out:
  20749. spin_unlock(&pwq->pool->lock);
  20750. + rcu_read_unlock();
  20751. }
  20752. /**
  20753. @@ -1478,14 +1510,14 @@
  20754. bool ret = false;
  20755. unsigned long flags;
  20756. - local_irq_save(flags);
  20757. + local_lock_irqsave(pendingb_lock,flags);
  20758. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  20759. __queue_work(cpu, wq, work);
  20760. ret = true;
  20761. }
  20762. - local_irq_restore(flags);
  20763. + local_unlock_irqrestore(pendingb_lock, flags);
  20764. return ret;
  20765. }
  20766. EXPORT_SYMBOL(queue_work_on);
  20767. @@ -1552,14 +1584,14 @@
  20768. unsigned long flags;
  20769. /* read the comment in __queue_work() */
  20770. - local_irq_save(flags);
  20771. + local_lock_irqsave(pendingb_lock, flags);
  20772. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  20773. __queue_delayed_work(cpu, wq, dwork, delay);
  20774. ret = true;
  20775. }
  20776. - local_irq_restore(flags);
  20777. + local_unlock_irqrestore(pendingb_lock, flags);
  20778. return ret;
  20779. }
  20780. EXPORT_SYMBOL(queue_delayed_work_on);
  20781. @@ -1594,7 +1626,7 @@
  20782. if (likely(ret >= 0)) {
  20783. __queue_delayed_work(cpu, wq, dwork, delay);
  20784. - local_irq_restore(flags);
  20785. + local_unlock_irqrestore(pendingb_lock, flags);
  20786. }
  20787. /* -ENOENT from try_to_grab_pending() becomes %true */
  20788. @@ -1627,7 +1659,9 @@
  20789. worker->last_active = jiffies;
  20790. /* idle_list is LIFO */
  20791. + rt_lock_idle_list(pool);
  20792. list_add(&worker->entry, &pool->idle_list);
  20793. + rt_unlock_idle_list(pool);
  20794. if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
  20795. mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
  20796. @@ -1660,7 +1694,9 @@
  20797. return;
  20798. worker_clr_flags(worker, WORKER_IDLE);
  20799. pool->nr_idle--;
  20800. + rt_lock_idle_list(pool);
  20801. list_del_init(&worker->entry);
  20802. + rt_unlock_idle_list(pool);
  20803. }
  20804. static struct worker *alloc_worker(int node)
  20805. @@ -1826,7 +1862,9 @@
  20806. pool->nr_workers--;
  20807. pool->nr_idle--;
  20808. + rt_lock_idle_list(pool);
  20809. list_del_init(&worker->entry);
  20810. + rt_unlock_idle_list(pool);
  20811. worker->flags |= WORKER_DIE;
  20812. wake_up_process(worker->task);
  20813. }
  20814. @@ -2785,14 +2823,14 @@
  20815. might_sleep();
  20816. - local_irq_disable();
  20817. + rcu_read_lock();
  20818. pool = get_work_pool(work);
  20819. if (!pool) {
  20820. - local_irq_enable();
  20821. + rcu_read_unlock();
  20822. return false;
  20823. }
  20824. - spin_lock(&pool->lock);
  20825. + spin_lock_irq(&pool->lock);
  20826. /* see the comment in try_to_grab_pending() with the same code */
  20827. pwq = get_work_pwq(work);
  20828. if (pwq) {
  20829. @@ -2821,10 +2859,11 @@
  20830. else
  20831. lock_map_acquire_read(&pwq->wq->lockdep_map);
  20832. lock_map_release(&pwq->wq->lockdep_map);
  20833. -
  20834. + rcu_read_unlock();
  20835. return true;
  20836. already_gone:
  20837. spin_unlock_irq(&pool->lock);
  20838. + rcu_read_unlock();
  20839. return false;
  20840. }
  20841. @@ -2911,7 +2950,7 @@
  20842. /* tell other tasks trying to grab @work to back off */
  20843. mark_work_canceling(work);
  20844. - local_irq_restore(flags);
  20845. + local_unlock_irqrestore(pendingb_lock, flags);
  20846. flush_work(work);
  20847. clear_work_data(work);
  20848. @@ -2966,10 +3005,10 @@
  20849. */
  20850. bool flush_delayed_work(struct delayed_work *dwork)
  20851. {
  20852. - local_irq_disable();
  20853. + local_lock_irq(pendingb_lock);
  20854. if (del_timer_sync(&dwork->timer))
  20855. __queue_work(dwork->cpu, dwork->wq, &dwork->work);
  20856. - local_irq_enable();
  20857. + local_unlock_irq(pendingb_lock);
  20858. return flush_work(&dwork->work);
  20859. }
  20860. EXPORT_SYMBOL(flush_delayed_work);
  20861. @@ -2987,7 +3026,7 @@
  20862. return false;
  20863. set_work_pool_and_clear_pending(work, get_work_pool_id(work));
  20864. - local_irq_restore(flags);
  20865. + local_unlock_irqrestore(pendingb_lock, flags);
  20866. return ret;
  20867. }
  20868. @@ -3245,7 +3284,7 @@
  20869. * put_unbound_pool - put a worker_pool
  20870. * @pool: worker_pool to put
  20871. *
  20872. - * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU
  20873. + * Put @pool. If its refcnt reaches zero, it gets destroyed in RCU
  20874. * safe manner. get_unbound_pool() calls this function on its failure path
  20875. * and this function should be able to release pools which went through,
  20876. * successfully or not, init_worker_pool().
  20877. @@ -3299,8 +3338,8 @@
  20878. del_timer_sync(&pool->idle_timer);
  20879. del_timer_sync(&pool->mayday_timer);
  20880. - /* sched-RCU protected to allow dereferences from get_work_pool() */
  20881. - call_rcu_sched(&pool->rcu, rcu_free_pool);
  20882. + /* RCU protected to allow dereferences from get_work_pool() */
  20883. + call_rcu(&pool->rcu, rcu_free_pool);
  20884. }
  20885. /**
  20886. @@ -3407,14 +3446,14 @@
  20887. put_unbound_pool(pool);
  20888. mutex_unlock(&wq_pool_mutex);
  20889. - call_rcu_sched(&pwq->rcu, rcu_free_pwq);
  20890. + call_rcu(&pwq->rcu, rcu_free_pwq);
  20891. /*
  20892. * If we're the last pwq going away, @wq is already dead and no one
  20893. * is gonna access it anymore. Schedule RCU free.
  20894. */
  20895. if (is_last)
  20896. - call_rcu_sched(&wq->rcu, rcu_free_wq);
  20897. + call_rcu(&wq->rcu, rcu_free_wq);
  20898. }
  20899. /**
  20900. @@ -4064,7 +4103,7 @@
  20901. * The base ref is never dropped on per-cpu pwqs. Directly
  20902. * schedule RCU free.
  20903. */
  20904. - call_rcu_sched(&wq->rcu, rcu_free_wq);
  20905. + call_rcu(&wq->rcu, rcu_free_wq);
  20906. } else {
  20907. /*
  20908. * We're the sole accessor of @wq at this point. Directly
  20909. @@ -4157,7 +4196,8 @@
  20910. struct pool_workqueue *pwq;
  20911. bool ret;
  20912. - rcu_read_lock_sched();
  20913. + rcu_read_lock();
  20914. + preempt_disable();
  20915. if (cpu == WORK_CPU_UNBOUND)
  20916. cpu = smp_processor_id();
  20917. @@ -4168,7 +4208,8 @@
  20918. pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
  20919. ret = !list_empty(&pwq->delayed_works);
  20920. - rcu_read_unlock_sched();
  20921. + preempt_enable();
  20922. + rcu_read_unlock();
  20923. return ret;
  20924. }
  20925. @@ -4194,15 +4235,15 @@
  20926. if (work_pending(work))
  20927. ret |= WORK_BUSY_PENDING;
  20928. - local_irq_save(flags);
  20929. + rcu_read_lock();
  20930. pool = get_work_pool(work);
  20931. if (pool) {
  20932. - spin_lock(&pool->lock);
  20933. + spin_lock_irqsave(&pool->lock, flags);
  20934. if (find_worker_executing_work(pool, work))
  20935. ret |= WORK_BUSY_RUNNING;
  20936. - spin_unlock(&pool->lock);
  20937. + spin_unlock_irqrestore(&pool->lock, flags);
  20938. }
  20939. - local_irq_restore(flags);
  20940. + rcu_read_unlock();
  20941. return ret;
  20942. }
  20943. @@ -4391,7 +4432,7 @@
  20944. unsigned long flags;
  20945. int pi;
  20946. - rcu_read_lock_sched();
  20947. + rcu_read_lock();
  20948. pr_info("Showing busy workqueues and worker pools:\n");
  20949. @@ -4444,7 +4485,7 @@
  20950. spin_unlock_irqrestore(&pool->lock, flags);
  20951. }
  20952. - rcu_read_unlock_sched();
  20953. + rcu_read_unlock();
  20954. }
  20955. /*
  20956. @@ -4782,16 +4823,16 @@
  20957. * nr_active is monotonically decreasing. It's safe
  20958. * to peek without lock.
  20959. */
  20960. - rcu_read_lock_sched();
  20961. + rcu_read_lock();
  20962. for_each_pwq(pwq, wq) {
  20963. WARN_ON_ONCE(pwq->nr_active < 0);
  20964. if (pwq->nr_active) {
  20965. busy = true;
  20966. - rcu_read_unlock_sched();
  20967. + rcu_read_unlock();
  20968. goto out_unlock;
  20969. }
  20970. }
  20971. - rcu_read_unlock_sched();
  20972. + rcu_read_unlock();
  20973. }
  20974. out_unlock:
  20975. mutex_unlock(&wq_pool_mutex);
  20976. @@ -4981,7 +5022,8 @@
  20977. const char *delim = "";
  20978. int node, written = 0;
  20979. - rcu_read_lock_sched();
  20980. + get_online_cpus();
  20981. + rcu_read_lock();
  20982. for_each_node(node) {
  20983. written += scnprintf(buf + written, PAGE_SIZE - written,
  20984. "%s%d:%d", delim, node,
  20985. @@ -4989,7 +5031,8 @@
  20986. delim = " ";
  20987. }
  20988. written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
  20989. - rcu_read_unlock_sched();
  20990. + rcu_read_unlock();
  20991. + put_online_cpus();
  20992. return written;
  20993. }
  20994. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/workqueue_internal.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/workqueue_internal.h
  20995. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/kernel/workqueue_internal.h 2017-04-16 10:38:30.000000000 +0200
  20996. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/kernel/workqueue_internal.h 2017-04-18 17:54:26.000000000 +0200
  20997. @@ -43,6 +43,7 @@
  20998. unsigned long last_active; /* L: last active timestamp */
  20999. unsigned int flags; /* X: flags */
  21000. int id; /* I: worker id */
  21001. + int sleeping; /* None */
  21002. /*
  21003. * Opaque string set with work_set_desc(). Printed out with task
  21004. @@ -68,7 +69,7 @@
  21005. * Scheduler hooks for concurrency managed workqueue. Only to be used from
  21006. * sched/core.c and workqueue.c.
  21007. */
  21008. -void wq_worker_waking_up(struct task_struct *task, int cpu);
  21009. -struct task_struct *wq_worker_sleeping(struct task_struct *task);
  21010. +void wq_worker_running(struct task_struct *task);
  21011. +void wq_worker_sleeping(struct task_struct *task);
  21012. #endif /* _KERNEL_WORKQUEUE_INTERNAL_H */
  21013. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/Kconfig
  21014. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/Kconfig 2017-04-16 10:38:30.000000000 +0200
  21015. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/Kconfig 2017-04-18 17:54:26.000000000 +0200
  21016. @@ -400,6 +400,7 @@
  21017. config CPUMASK_OFFSTACK
  21018. bool "Force CPU masks off stack" if DEBUG_PER_CPU_MAPS
  21019. + depends on !PREEMPT_RT_FULL
  21020. help
  21021. Use dynamic allocation for cpumask_var_t, instead of putting
  21022. them on the stack. This is a bit more expensive, but avoids
  21023. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/debugobjects.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/debugobjects.c
  21024. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/debugobjects.c 2017-04-16 10:38:30.000000000 +0200
  21025. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/debugobjects.c 2017-04-18 17:54:26.000000000 +0200
  21026. @@ -308,7 +308,10 @@
  21027. struct debug_obj *obj;
  21028. unsigned long flags;
  21029. - fill_pool();
  21030. +#ifdef CONFIG_PREEMPT_RT_FULL
  21031. + if (preempt_count() == 0 && !irqs_disabled())
  21032. +#endif
  21033. + fill_pool();
  21034. db = get_bucket((unsigned long) addr);
  21035. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/idr.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/idr.c
  21036. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/idr.c 2017-04-16 10:38:30.000000000 +0200
  21037. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/idr.c 2017-04-18 17:54:26.000000000 +0200
  21038. @@ -30,6 +30,7 @@
  21039. #include <linux/idr.h>
  21040. #include <linux/spinlock.h>
  21041. #include <linux/percpu.h>
  21042. +#include <linux/locallock.h>
  21043. #define MAX_IDR_SHIFT (sizeof(int) * 8 - 1)
  21044. #define MAX_IDR_BIT (1U << MAX_IDR_SHIFT)
  21045. @@ -45,6 +46,37 @@
  21046. static DEFINE_PER_CPU(int, idr_preload_cnt);
  21047. static DEFINE_SPINLOCK(simple_ida_lock);
  21048. +#ifdef CONFIG_PREEMPT_RT_FULL
  21049. +static DEFINE_LOCAL_IRQ_LOCK(idr_lock);
  21050. +
  21051. +static inline void idr_preload_lock(void)
  21052. +{
  21053. + local_lock(idr_lock);
  21054. +}
  21055. +
  21056. +static inline void idr_preload_unlock(void)
  21057. +{
  21058. + local_unlock(idr_lock);
  21059. +}
  21060. +
  21061. +void idr_preload_end(void)
  21062. +{
  21063. + idr_preload_unlock();
  21064. +}
  21065. +EXPORT_SYMBOL(idr_preload_end);
  21066. +#else
  21067. +static inline void idr_preload_lock(void)
  21068. +{
  21069. + preempt_disable();
  21070. +}
  21071. +
  21072. +static inline void idr_preload_unlock(void)
  21073. +{
  21074. + preempt_enable();
  21075. +}
  21076. +#endif
  21077. +
  21078. +
  21079. /* the maximum ID which can be allocated given idr->layers */
  21080. static int idr_max(int layers)
  21081. {
  21082. @@ -115,14 +147,14 @@
  21083. * context. See idr_preload() for details.
  21084. */
  21085. if (!in_interrupt()) {
  21086. - preempt_disable();
  21087. + idr_preload_lock();
  21088. new = __this_cpu_read(idr_preload_head);
  21089. if (new) {
  21090. __this_cpu_write(idr_preload_head, new->ary[0]);
  21091. __this_cpu_dec(idr_preload_cnt);
  21092. new->ary[0] = NULL;
  21093. }
  21094. - preempt_enable();
  21095. + idr_preload_unlock();
  21096. if (new)
  21097. return new;
  21098. }
  21099. @@ -366,7 +398,6 @@
  21100. idr_mark_full(pa, id);
  21101. }
  21102. -
  21103. /**
  21104. * idr_preload - preload for idr_alloc()
  21105. * @gfp_mask: allocation mask to use for preloading
  21106. @@ -401,7 +432,7 @@
  21107. WARN_ON_ONCE(in_interrupt());
  21108. might_sleep_if(gfpflags_allow_blocking(gfp_mask));
  21109. - preempt_disable();
  21110. + idr_preload_lock();
  21111. /*
  21112. * idr_alloc() is likely to succeed w/o full idr_layer buffer and
  21113. @@ -413,9 +444,9 @@
  21114. while (__this_cpu_read(idr_preload_cnt) < MAX_IDR_FREE) {
  21115. struct idr_layer *new;
  21116. - preempt_enable();
  21117. + idr_preload_unlock();
  21118. new = kmem_cache_zalloc(idr_layer_cache, gfp_mask);
  21119. - preempt_disable();
  21120. + idr_preload_lock();
  21121. if (!new)
  21122. break;
  21123. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/irq_poll.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/irq_poll.c
  21124. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/irq_poll.c 2017-04-16 10:38:30.000000000 +0200
  21125. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/irq_poll.c 2017-04-18 17:54:26.000000000 +0200
  21126. @@ -36,6 +36,7 @@
  21127. list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll));
  21128. __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
  21129. local_irq_restore(flags);
  21130. + preempt_check_resched_rt();
  21131. }
  21132. EXPORT_SYMBOL(irq_poll_sched);
  21133. @@ -71,6 +72,7 @@
  21134. local_irq_save(flags);
  21135. __irq_poll_complete(iop);
  21136. local_irq_restore(flags);
  21137. + preempt_check_resched_rt();
  21138. }
  21139. EXPORT_SYMBOL(irq_poll_complete);
  21140. @@ -95,6 +97,7 @@
  21141. }
  21142. local_irq_enable();
  21143. + preempt_check_resched_rt();
  21144. /* Even though interrupts have been re-enabled, this
  21145. * access is safe because interrupts can only add new
  21146. @@ -132,6 +135,7 @@
  21147. __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
  21148. local_irq_enable();
  21149. + preempt_check_resched_rt();
  21150. }
  21151. /**
  21152. @@ -195,6 +199,7 @@
  21153. this_cpu_ptr(&blk_cpu_iopoll));
  21154. __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
  21155. local_irq_enable();
  21156. + preempt_check_resched_rt();
  21157. return 0;
  21158. }
  21159. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/locking-selftest.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/locking-selftest.c
  21160. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/locking-selftest.c 2017-04-16 10:38:30.000000000 +0200
  21161. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/locking-selftest.c 2017-04-18 17:54:26.000000000 +0200
  21162. @@ -590,6 +590,8 @@
  21163. #include "locking-selftest-spin-hardirq.h"
  21164. GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin)
  21165. +#ifndef CONFIG_PREEMPT_RT_FULL
  21166. +
  21167. #include "locking-selftest-rlock-hardirq.h"
  21168. GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_rlock)
  21169. @@ -605,9 +607,12 @@
  21170. #include "locking-selftest-wlock-softirq.h"
  21171. GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_wlock)
  21172. +#endif
  21173. +
  21174. #undef E1
  21175. #undef E2
  21176. +#ifndef CONFIG_PREEMPT_RT_FULL
  21177. /*
  21178. * Enabling hardirqs with a softirq-safe lock held:
  21179. */
  21180. @@ -640,6 +645,8 @@
  21181. #undef E1
  21182. #undef E2
  21183. +#endif
  21184. +
  21185. /*
  21186. * Enabling irqs with an irq-safe lock held:
  21187. */
  21188. @@ -663,6 +670,8 @@
  21189. #include "locking-selftest-spin-hardirq.h"
  21190. GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_spin)
  21191. +#ifndef CONFIG_PREEMPT_RT_FULL
  21192. +
  21193. #include "locking-selftest-rlock-hardirq.h"
  21194. GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_rlock)
  21195. @@ -678,6 +687,8 @@
  21196. #include "locking-selftest-wlock-softirq.h"
  21197. GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock)
  21198. +#endif
  21199. +
  21200. #undef E1
  21201. #undef E2
  21202. @@ -709,6 +720,8 @@
  21203. #include "locking-selftest-spin-hardirq.h"
  21204. GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_spin)
  21205. +#ifndef CONFIG_PREEMPT_RT_FULL
  21206. +
  21207. #include "locking-selftest-rlock-hardirq.h"
  21208. GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_rlock)
  21209. @@ -724,6 +737,8 @@
  21210. #include "locking-selftest-wlock-softirq.h"
  21211. GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock)
  21212. +#endif
  21213. +
  21214. #undef E1
  21215. #undef E2
  21216. #undef E3
  21217. @@ -757,6 +772,8 @@
  21218. #include "locking-selftest-spin-hardirq.h"
  21219. GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_spin)
  21220. +#ifndef CONFIG_PREEMPT_RT_FULL
  21221. +
  21222. #include "locking-selftest-rlock-hardirq.h"
  21223. GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_rlock)
  21224. @@ -772,10 +789,14 @@
  21225. #include "locking-selftest-wlock-softirq.h"
  21226. GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_wlock)
  21227. +#endif
  21228. +
  21229. #undef E1
  21230. #undef E2
  21231. #undef E3
  21232. +#ifndef CONFIG_PREEMPT_RT_FULL
  21233. +
  21234. /*
  21235. * read-lock / write-lock irq inversion.
  21236. *
  21237. @@ -838,6 +859,10 @@
  21238. #undef E2
  21239. #undef E3
  21240. +#endif
  21241. +
  21242. +#ifndef CONFIG_PREEMPT_RT_FULL
  21243. +
  21244. /*
  21245. * read-lock / write-lock recursion that is actually safe.
  21246. */
  21247. @@ -876,6 +901,8 @@
  21248. #undef E2
  21249. #undef E3
  21250. +#endif
  21251. +
  21252. /*
  21253. * read-lock / write-lock recursion that is unsafe.
  21254. */
  21255. @@ -1858,6 +1885,7 @@
  21256. printk(" --------------------------------------------------------------------------\n");
  21257. +#ifndef CONFIG_PREEMPT_RT_FULL
  21258. /*
  21259. * irq-context testcases:
  21260. */
  21261. @@ -1870,6 +1898,28 @@
  21262. DO_TESTCASE_6x2("irq read-recursion", irq_read_recursion);
  21263. // DO_TESTCASE_6x2B("irq read-recursion #2", irq_read_recursion2);
  21264. +#else
  21265. + /* On -rt, we only do hardirq context test for raw spinlock */
  21266. + DO_TESTCASE_1B("hard-irqs-on + irq-safe-A", irqsafe1_hard_spin, 12);
  21267. + DO_TESTCASE_1B("hard-irqs-on + irq-safe-A", irqsafe1_hard_spin, 21);
  21268. +
  21269. + DO_TESTCASE_1B("hard-safe-A + irqs-on", irqsafe2B_hard_spin, 12);
  21270. + DO_TESTCASE_1B("hard-safe-A + irqs-on", irqsafe2B_hard_spin, 21);
  21271. +
  21272. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 123);
  21273. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 132);
  21274. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 213);
  21275. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 231);
  21276. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 312);
  21277. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 321);
  21278. +
  21279. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 123);
  21280. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 132);
  21281. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 213);
  21282. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 231);
  21283. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 312);
  21284. + DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 321);
  21285. +#endif
  21286. ww_tests();
  21287. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/percpu_ida.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/percpu_ida.c
  21288. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/percpu_ida.c 2017-04-16 10:38:30.000000000 +0200
  21289. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/percpu_ida.c 2017-04-18 17:54:26.000000000 +0200
  21290. @@ -26,6 +26,9 @@
  21291. #include <linux/string.h>
  21292. #include <linux/spinlock.h>
  21293. #include <linux/percpu_ida.h>
  21294. +#include <linux/locallock.h>
  21295. +
  21296. +static DEFINE_LOCAL_IRQ_LOCK(irq_off_lock);
  21297. struct percpu_ida_cpu {
  21298. /*
  21299. @@ -148,13 +151,13 @@
  21300. unsigned long flags;
  21301. int tag;
  21302. - local_irq_save(flags);
  21303. + local_lock_irqsave(irq_off_lock, flags);
  21304. tags = this_cpu_ptr(pool->tag_cpu);
  21305. /* Fastpath */
  21306. tag = alloc_local_tag(tags);
  21307. if (likely(tag >= 0)) {
  21308. - local_irq_restore(flags);
  21309. + local_unlock_irqrestore(irq_off_lock, flags);
  21310. return tag;
  21311. }
  21312. @@ -173,6 +176,7 @@
  21313. if (!tags->nr_free)
  21314. alloc_global_tags(pool, tags);
  21315. +
  21316. if (!tags->nr_free)
  21317. steal_tags(pool, tags);
  21318. @@ -184,7 +188,7 @@
  21319. }
  21320. spin_unlock(&pool->lock);
  21321. - local_irq_restore(flags);
  21322. + local_unlock_irqrestore(irq_off_lock, flags);
  21323. if (tag >= 0 || state == TASK_RUNNING)
  21324. break;
  21325. @@ -196,7 +200,7 @@
  21326. schedule();
  21327. - local_irq_save(flags);
  21328. + local_lock_irqsave(irq_off_lock, flags);
  21329. tags = this_cpu_ptr(pool->tag_cpu);
  21330. }
  21331. if (state != TASK_RUNNING)
  21332. @@ -221,7 +225,7 @@
  21333. BUG_ON(tag >= pool->nr_tags);
  21334. - local_irq_save(flags);
  21335. + local_lock_irqsave(irq_off_lock, flags);
  21336. tags = this_cpu_ptr(pool->tag_cpu);
  21337. spin_lock(&tags->lock);
  21338. @@ -253,7 +257,7 @@
  21339. spin_unlock(&pool->lock);
  21340. }
  21341. - local_irq_restore(flags);
  21342. + local_unlock_irqrestore(irq_off_lock, flags);
  21343. }
  21344. EXPORT_SYMBOL_GPL(percpu_ida_free);
  21345. @@ -345,7 +349,7 @@
  21346. struct percpu_ida_cpu *remote;
  21347. unsigned cpu, i, err = 0;
  21348. - local_irq_save(flags);
  21349. + local_lock_irqsave(irq_off_lock, flags);
  21350. for_each_possible_cpu(cpu) {
  21351. remote = per_cpu_ptr(pool->tag_cpu, cpu);
  21352. spin_lock(&remote->lock);
  21353. @@ -367,7 +371,7 @@
  21354. }
  21355. spin_unlock(&pool->lock);
  21356. out:
  21357. - local_irq_restore(flags);
  21358. + local_unlock_irqrestore(irq_off_lock, flags);
  21359. return err;
  21360. }
  21361. EXPORT_SYMBOL_GPL(percpu_ida_for_each_free);
  21362. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/radix-tree.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/radix-tree.c
  21363. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/radix-tree.c 2017-04-16 10:38:30.000000000 +0200
  21364. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/radix-tree.c 2017-04-18 17:54:26.000000000 +0200
  21365. @@ -36,7 +36,7 @@
  21366. #include <linux/bitops.h>
  21367. #include <linux/rcupdate.h>
  21368. #include <linux/preempt.h> /* in_interrupt() */
  21369. -
  21370. +#include <linux/locallock.h>
  21371. /* Number of nodes in fully populated tree of given height */
  21372. static unsigned long height_to_maxnodes[RADIX_TREE_MAX_PATH + 1] __read_mostly;
  21373. @@ -68,6 +68,7 @@
  21374. struct radix_tree_node *nodes;
  21375. };
  21376. static DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = { 0, };
  21377. +static DEFINE_LOCAL_IRQ_LOCK(radix_tree_preloads_lock);
  21378. static inline void *node_to_entry(void *ptr)
  21379. {
  21380. @@ -290,13 +291,14 @@
  21381. * succeed in getting a node here (and never reach
  21382. * kmem_cache_alloc)
  21383. */
  21384. - rtp = this_cpu_ptr(&radix_tree_preloads);
  21385. + rtp = &get_locked_var(radix_tree_preloads_lock, radix_tree_preloads);
  21386. if (rtp->nr) {
  21387. ret = rtp->nodes;
  21388. rtp->nodes = ret->private_data;
  21389. ret->private_data = NULL;
  21390. rtp->nr--;
  21391. }
  21392. + put_locked_var(radix_tree_preloads_lock, radix_tree_preloads);
  21393. /*
  21394. * Update the allocation stack trace as this is more useful
  21395. * for debugging.
  21396. @@ -357,14 +359,14 @@
  21397. */
  21398. gfp_mask &= ~__GFP_ACCOUNT;
  21399. - preempt_disable();
  21400. + local_lock(radix_tree_preloads_lock);
  21401. rtp = this_cpu_ptr(&radix_tree_preloads);
  21402. while (rtp->nr < nr) {
  21403. - preempt_enable();
  21404. + local_unlock(radix_tree_preloads_lock);
  21405. node = kmem_cache_alloc(radix_tree_node_cachep, gfp_mask);
  21406. if (node == NULL)
  21407. goto out;
  21408. - preempt_disable();
  21409. + local_lock(radix_tree_preloads_lock);
  21410. rtp = this_cpu_ptr(&radix_tree_preloads);
  21411. if (rtp->nr < nr) {
  21412. node->private_data = rtp->nodes;
  21413. @@ -406,7 +408,7 @@
  21414. if (gfpflags_allow_blocking(gfp_mask))
  21415. return __radix_tree_preload(gfp_mask, RADIX_TREE_PRELOAD_SIZE);
  21416. /* Preloading doesn't help anything with this gfp mask, skip it */
  21417. - preempt_disable();
  21418. + local_lock(radix_tree_preloads_lock);
  21419. return 0;
  21420. }
  21421. EXPORT_SYMBOL(radix_tree_maybe_preload);
  21422. @@ -422,7 +424,7 @@
  21423. /* Preloading doesn't help anything with this gfp mask, skip it */
  21424. if (!gfpflags_allow_blocking(gfp_mask)) {
  21425. - preempt_disable();
  21426. + local_lock(radix_tree_preloads_lock);
  21427. return 0;
  21428. }
  21429. @@ -456,6 +458,12 @@
  21430. return __radix_tree_preload(gfp_mask, nr_nodes);
  21431. }
  21432. +void radix_tree_preload_end(void)
  21433. +{
  21434. + local_unlock(radix_tree_preloads_lock);
  21435. +}
  21436. +EXPORT_SYMBOL(radix_tree_preload_end);
  21437. +
  21438. /*
  21439. * The maximum index which can be stored in a radix tree
  21440. */
  21441. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/scatterlist.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/scatterlist.c
  21442. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/scatterlist.c 2017-04-16 10:38:30.000000000 +0200
  21443. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/scatterlist.c 2017-04-18 17:54:26.000000000 +0200
  21444. @@ -620,7 +620,7 @@
  21445. flush_kernel_dcache_page(miter->page);
  21446. if (miter->__flags & SG_MITER_ATOMIC) {
  21447. - WARN_ON_ONCE(preemptible());
  21448. + WARN_ON_ONCE(!pagefault_disabled());
  21449. kunmap_atomic(miter->addr);
  21450. } else
  21451. kunmap(miter->page);
  21452. @@ -664,7 +664,7 @@
  21453. if (!sg_miter_skip(&miter, skip))
  21454. return false;
  21455. - local_irq_save(flags);
  21456. + local_irq_save_nort(flags);
  21457. while (sg_miter_next(&miter) && offset < buflen) {
  21458. unsigned int len;
  21459. @@ -681,7 +681,7 @@
  21460. sg_miter_stop(&miter);
  21461. - local_irq_restore(flags);
  21462. + local_irq_restore_nort(flags);
  21463. return offset;
  21464. }
  21465. EXPORT_SYMBOL(sg_copy_buffer);
  21466. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/smp_processor_id.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/smp_processor_id.c
  21467. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/lib/smp_processor_id.c 2017-04-16 10:38:30.000000000 +0200
  21468. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/lib/smp_processor_id.c 2017-04-18 17:54:26.000000000 +0200
  21469. @@ -39,8 +39,9 @@
  21470. if (!printk_ratelimit())
  21471. goto out_enable;
  21472. - printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x] code: %s/%d\n",
  21473. - what1, what2, preempt_count() - 1, current->comm, current->pid);
  21474. + printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x %08x] code: %s/%d\n",
  21475. + what1, what2, preempt_count() - 1, __migrate_disabled(current),
  21476. + current->comm, current->pid);
  21477. print_symbol("caller is %s\n", (long)__builtin_return_address(0));
  21478. dump_stack();
  21479. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/Kconfig linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/Kconfig
  21480. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/Kconfig 2017-04-16 10:38:30.000000000 +0200
  21481. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/Kconfig 2017-04-18 17:54:26.000000000 +0200
  21482. @@ -410,7 +410,7 @@
  21483. config TRANSPARENT_HUGEPAGE
  21484. bool "Transparent Hugepage Support"
  21485. - depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE
  21486. + depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE && !PREEMPT_RT_FULL
  21487. select COMPACTION
  21488. select RADIX_TREE_MULTIORDER
  21489. help
  21490. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/backing-dev.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/backing-dev.c
  21491. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/backing-dev.c 2017-04-16 10:38:30.000000000 +0200
  21492. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/backing-dev.c 2017-04-18 17:54:26.000000000 +0200
  21493. @@ -457,9 +457,9 @@
  21494. {
  21495. unsigned long flags;
  21496. - local_irq_save(flags);
  21497. + local_irq_save_nort(flags);
  21498. if (!atomic_dec_and_lock(&congested->refcnt, &cgwb_lock)) {
  21499. - local_irq_restore(flags);
  21500. + local_irq_restore_nort(flags);
  21501. return;
  21502. }
  21503. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/compaction.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/compaction.c
  21504. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/compaction.c 2017-04-16 10:38:30.000000000 +0200
  21505. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/compaction.c 2017-04-18 17:54:26.000000000 +0200
  21506. @@ -1593,10 +1593,12 @@
  21507. block_start_pfn(cc->migrate_pfn, cc->order);
  21508. if (cc->last_migrated_pfn < current_block_start) {
  21509. - cpu = get_cpu();
  21510. + cpu = get_cpu_light();
  21511. + local_lock_irq(swapvec_lock);
  21512. lru_add_drain_cpu(cpu);
  21513. + local_unlock_irq(swapvec_lock);
  21514. drain_local_pages(zone);
  21515. - put_cpu();
  21516. + put_cpu_light();
  21517. /* No more flushing until we migrate again */
  21518. cc->last_migrated_pfn = 0;
  21519. }
  21520. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/filemap.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/filemap.c
  21521. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/filemap.c 2017-04-16 10:38:30.000000000 +0200
  21522. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/filemap.c 2017-04-18 17:54:26.000000000 +0200
  21523. @@ -159,9 +159,12 @@
  21524. * node->private_list is protected by
  21525. * mapping->tree_lock.
  21526. */
  21527. - if (!list_empty(&node->private_list))
  21528. - list_lru_del(&workingset_shadow_nodes,
  21529. + if (!list_empty(&node->private_list)) {
  21530. + local_lock(workingset_shadow_lock);
  21531. + list_lru_del(&__workingset_shadow_nodes,
  21532. &node->private_list);
  21533. + local_unlock(workingset_shadow_lock);
  21534. + }
  21535. }
  21536. return 0;
  21537. }
  21538. @@ -217,8 +220,10 @@
  21539. if (!dax_mapping(mapping) && !workingset_node_pages(node) &&
  21540. list_empty(&node->private_list)) {
  21541. node->private_data = mapping;
  21542. - list_lru_add(&workingset_shadow_nodes,
  21543. - &node->private_list);
  21544. + local_lock(workingset_shadow_lock);
  21545. + list_lru_add(&__workingset_shadow_nodes,
  21546. + &node->private_list);
  21547. + local_unlock(workingset_shadow_lock);
  21548. }
  21549. }
  21550. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/highmem.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/highmem.c
  21551. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/highmem.c 2017-04-16 10:38:30.000000000 +0200
  21552. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/highmem.c 2017-04-18 17:54:26.000000000 +0200
  21553. @@ -29,10 +29,11 @@
  21554. #include <linux/kgdb.h>
  21555. #include <asm/tlbflush.h>
  21556. -
  21557. +#ifndef CONFIG_PREEMPT_RT_FULL
  21558. #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32)
  21559. DEFINE_PER_CPU(int, __kmap_atomic_idx);
  21560. #endif
  21561. +#endif
  21562. /*
  21563. * Virtual_count is not a pure "count".
  21564. @@ -107,8 +108,9 @@
  21565. unsigned long totalhigh_pages __read_mostly;
  21566. EXPORT_SYMBOL(totalhigh_pages);
  21567. -
  21568. +#ifndef CONFIG_PREEMPT_RT_FULL
  21569. EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx);
  21570. +#endif
  21571. unsigned int nr_free_highpages (void)
  21572. {
  21573. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/memcontrol.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/memcontrol.c
  21574. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/memcontrol.c 2017-04-16 10:38:30.000000000 +0200
  21575. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/memcontrol.c 2017-04-18 17:54:26.000000000 +0200
  21576. @@ -67,6 +67,7 @@
  21577. #include <net/sock.h>
  21578. #include <net/ip.h>
  21579. #include "slab.h"
  21580. +#include <linux/locallock.h>
  21581. #include <asm/uaccess.h>
  21582. @@ -92,6 +93,8 @@
  21583. #define do_swap_account 0
  21584. #endif
  21585. +static DEFINE_LOCAL_IRQ_LOCK(event_lock);
  21586. +
  21587. /* Whether legacy memory+swap accounting is active */
  21588. static bool do_memsw_account(void)
  21589. {
  21590. @@ -1692,6 +1695,7 @@
  21591. #define FLUSHING_CACHED_CHARGE 0
  21592. };
  21593. static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
  21594. +static DEFINE_LOCAL_IRQ_LOCK(memcg_stock_ll);
  21595. static DEFINE_MUTEX(percpu_charge_mutex);
  21596. /**
  21597. @@ -1714,7 +1718,7 @@
  21598. if (nr_pages > CHARGE_BATCH)
  21599. return ret;
  21600. - local_irq_save(flags);
  21601. + local_lock_irqsave(memcg_stock_ll, flags);
  21602. stock = this_cpu_ptr(&memcg_stock);
  21603. if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
  21604. @@ -1722,7 +1726,7 @@
  21605. ret = true;
  21606. }
  21607. - local_irq_restore(flags);
  21608. + local_unlock_irqrestore(memcg_stock_ll, flags);
  21609. return ret;
  21610. }
  21611. @@ -1749,13 +1753,13 @@
  21612. struct memcg_stock_pcp *stock;
  21613. unsigned long flags;
  21614. - local_irq_save(flags);
  21615. + local_lock_irqsave(memcg_stock_ll, flags);
  21616. stock = this_cpu_ptr(&memcg_stock);
  21617. drain_stock(stock);
  21618. clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
  21619. - local_irq_restore(flags);
  21620. + local_unlock_irqrestore(memcg_stock_ll, flags);
  21621. }
  21622. /*
  21623. @@ -1767,7 +1771,7 @@
  21624. struct memcg_stock_pcp *stock;
  21625. unsigned long flags;
  21626. - local_irq_save(flags);
  21627. + local_lock_irqsave(memcg_stock_ll, flags);
  21628. stock = this_cpu_ptr(&memcg_stock);
  21629. if (stock->cached != memcg) { /* reset if necessary */
  21630. @@ -1776,7 +1780,7 @@
  21631. }
  21632. stock->nr_pages += nr_pages;
  21633. - local_irq_restore(flags);
  21634. + local_unlock_irqrestore(memcg_stock_ll, flags);
  21635. }
  21636. /*
  21637. @@ -1792,7 +1796,7 @@
  21638. return;
  21639. /* Notify other cpus that system-wide "drain" is running */
  21640. get_online_cpus();
  21641. - curcpu = get_cpu();
  21642. + curcpu = get_cpu_light();
  21643. for_each_online_cpu(cpu) {
  21644. struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
  21645. struct mem_cgroup *memcg;
  21646. @@ -1809,7 +1813,7 @@
  21647. schedule_work_on(cpu, &stock->work);
  21648. }
  21649. }
  21650. - put_cpu();
  21651. + put_cpu_light();
  21652. put_online_cpus();
  21653. mutex_unlock(&percpu_charge_mutex);
  21654. }
  21655. @@ -4553,12 +4557,12 @@
  21656. ret = 0;
  21657. - local_irq_disable();
  21658. + local_lock_irq(event_lock);
  21659. mem_cgroup_charge_statistics(to, page, compound, nr_pages);
  21660. memcg_check_events(to, page);
  21661. mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
  21662. memcg_check_events(from, page);
  21663. - local_irq_enable();
  21664. + local_unlock_irq(event_lock);
  21665. out_unlock:
  21666. unlock_page(page);
  21667. out:
  21668. @@ -5433,10 +5437,10 @@
  21669. commit_charge(page, memcg, lrucare);
  21670. - local_irq_disable();
  21671. + local_lock_irq(event_lock);
  21672. mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
  21673. memcg_check_events(memcg, page);
  21674. - local_irq_enable();
  21675. + local_unlock_irq(event_lock);
  21676. if (do_memsw_account() && PageSwapCache(page)) {
  21677. swp_entry_t entry = { .val = page_private(page) };
  21678. @@ -5492,14 +5496,14 @@
  21679. memcg_oom_recover(memcg);
  21680. }
  21681. - local_irq_save(flags);
  21682. + local_lock_irqsave(event_lock, flags);
  21683. __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon);
  21684. __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file);
  21685. __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge);
  21686. __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout);
  21687. __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
  21688. memcg_check_events(memcg, dummy_page);
  21689. - local_irq_restore(flags);
  21690. + local_unlock_irqrestore(event_lock, flags);
  21691. if (!mem_cgroup_is_root(memcg))
  21692. css_put_many(&memcg->css, nr_pages);
  21693. @@ -5654,10 +5658,10 @@
  21694. commit_charge(newpage, memcg, false);
  21695. - local_irq_save(flags);
  21696. + local_lock_irqsave(event_lock, flags);
  21697. mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);
  21698. memcg_check_events(memcg, newpage);
  21699. - local_irq_restore(flags);
  21700. + local_unlock_irqrestore(event_lock, flags);
  21701. }
  21702. DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
  21703. @@ -5837,6 +5841,7 @@
  21704. {
  21705. struct mem_cgroup *memcg, *swap_memcg;
  21706. unsigned short oldid;
  21707. + unsigned long flags;
  21708. VM_BUG_ON_PAGE(PageLRU(page), page);
  21709. VM_BUG_ON_PAGE(page_count(page), page);
  21710. @@ -5877,12 +5882,16 @@
  21711. * important here to have the interrupts disabled because it is the
  21712. * only synchronisation we have for udpating the per-CPU variables.
  21713. */
  21714. + local_lock_irqsave(event_lock, flags);
  21715. +#ifndef CONFIG_PREEMPT_RT_BASE
  21716. VM_BUG_ON(!irqs_disabled());
  21717. +#endif
  21718. mem_cgroup_charge_statistics(memcg, page, false, -1);
  21719. memcg_check_events(memcg, page);
  21720. if (!mem_cgroup_is_root(memcg))
  21721. css_put(&memcg->css);
  21722. + local_unlock_irqrestore(event_lock, flags);
  21723. }
  21724. /*
  21725. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/mmu_context.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/mmu_context.c
  21726. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/mmu_context.c 2017-04-16 10:38:30.000000000 +0200
  21727. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/mmu_context.c 2017-04-18 17:54:26.000000000 +0200
  21728. @@ -23,6 +23,7 @@
  21729. struct task_struct *tsk = current;
  21730. task_lock(tsk);
  21731. + preempt_disable_rt();
  21732. active_mm = tsk->active_mm;
  21733. if (active_mm != mm) {
  21734. atomic_inc(&mm->mm_count);
  21735. @@ -30,6 +31,7 @@
  21736. }
  21737. tsk->mm = mm;
  21738. switch_mm(active_mm, mm, tsk);
  21739. + preempt_enable_rt();
  21740. task_unlock(tsk);
  21741. #ifdef finish_arch_post_lock_switch
  21742. finish_arch_post_lock_switch();
  21743. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/page_alloc.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/page_alloc.c
  21744. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/page_alloc.c 2017-04-16 10:38:30.000000000 +0200
  21745. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/page_alloc.c 2017-04-18 17:54:26.000000000 +0200
  21746. @@ -61,6 +61,7 @@
  21747. #include <linux/page_ext.h>
  21748. #include <linux/hugetlb.h>
  21749. #include <linux/sched/rt.h>
  21750. +#include <linux/locallock.h>
  21751. #include <linux/page_owner.h>
  21752. #include <linux/kthread.h>
  21753. #include <linux/memcontrol.h>
  21754. @@ -281,6 +282,18 @@
  21755. EXPORT_SYMBOL(nr_online_nodes);
  21756. #endif
  21757. +static DEFINE_LOCAL_IRQ_LOCK(pa_lock);
  21758. +
  21759. +#ifdef CONFIG_PREEMPT_RT_BASE
  21760. +# define cpu_lock_irqsave(cpu, flags) \
  21761. + local_lock_irqsave_on(pa_lock, flags, cpu)
  21762. +# define cpu_unlock_irqrestore(cpu, flags) \
  21763. + local_unlock_irqrestore_on(pa_lock, flags, cpu)
  21764. +#else
  21765. +# define cpu_lock_irqsave(cpu, flags) local_irq_save(flags)
  21766. +# define cpu_unlock_irqrestore(cpu, flags) local_irq_restore(flags)
  21767. +#endif
  21768. +
  21769. int page_group_by_mobility_disabled __read_mostly;
  21770. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  21771. @@ -1072,7 +1085,7 @@
  21772. #endif /* CONFIG_DEBUG_VM */
  21773. /*
  21774. - * Frees a number of pages from the PCP lists
  21775. + * Frees a number of pages which have been collected from the pcp lists.
  21776. * Assumes all pages on list are in same zone, and of same order.
  21777. * count is the number of pages to free.
  21778. *
  21779. @@ -1083,19 +1096,58 @@
  21780. * pinned" detection logic.
  21781. */
  21782. static void free_pcppages_bulk(struct zone *zone, int count,
  21783. - struct per_cpu_pages *pcp)
  21784. + struct list_head *list)
  21785. {
  21786. - int migratetype = 0;
  21787. - int batch_free = 0;
  21788. unsigned long nr_scanned;
  21789. bool isolated_pageblocks;
  21790. + unsigned long flags;
  21791. +
  21792. + spin_lock_irqsave(&zone->lock, flags);
  21793. - spin_lock(&zone->lock);
  21794. isolated_pageblocks = has_isolate_pageblock(zone);
  21795. nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
  21796. if (nr_scanned)
  21797. __mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned);
  21798. + while (!list_empty(list)) {
  21799. + struct page *page;
  21800. + int mt; /* migratetype of the to-be-freed page */
  21801. +
  21802. + page = list_first_entry(list, struct page, lru);
  21803. + /* must delete as __free_one_page list manipulates */
  21804. + list_del(&page->lru);
  21805. +
  21806. + mt = get_pcppage_migratetype(page);
  21807. + /* MIGRATE_ISOLATE page should not go to pcplists */
  21808. + VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
  21809. + /* Pageblock could have been isolated meanwhile */
  21810. + if (unlikely(isolated_pageblocks))
  21811. + mt = get_pageblock_migratetype(page);
  21812. +
  21813. + if (bulkfree_pcp_prepare(page))
  21814. + continue;
  21815. +
  21816. + __free_one_page(page, page_to_pfn(page), zone, 0, mt);
  21817. + trace_mm_page_pcpu_drain(page, 0, mt);
  21818. + count--;
  21819. + }
  21820. + WARN_ON(count != 0);
  21821. + spin_unlock_irqrestore(&zone->lock, flags);
  21822. +}
  21823. +
  21824. +/*
  21825. + * Moves a number of pages from the PCP lists to free list which
  21826. + * is freed outside of the locked region.
  21827. + *
  21828. + * Assumes all pages on list are in same zone, and of same order.
  21829. + * count is the number of pages to free.
  21830. + */
  21831. +static void isolate_pcp_pages(int count, struct per_cpu_pages *src,
  21832. + struct list_head *dst)
  21833. +{
  21834. + int migratetype = 0;
  21835. + int batch_free = 0;
  21836. +
  21837. while (count) {
  21838. struct page *page;
  21839. struct list_head *list;
  21840. @@ -1111,7 +1163,7 @@
  21841. batch_free++;
  21842. if (++migratetype == MIGRATE_PCPTYPES)
  21843. migratetype = 0;
  21844. - list = &pcp->lists[migratetype];
  21845. + list = &src->lists[migratetype];
  21846. } while (list_empty(list));
  21847. /* This is the only non-empty list. Free them all. */
  21848. @@ -1119,27 +1171,12 @@
  21849. batch_free = count;
  21850. do {
  21851. - int mt; /* migratetype of the to-be-freed page */
  21852. -
  21853. page = list_last_entry(list, struct page, lru);
  21854. - /* must delete as __free_one_page list manipulates */
  21855. list_del(&page->lru);
  21856. - mt = get_pcppage_migratetype(page);
  21857. - /* MIGRATE_ISOLATE page should not go to pcplists */
  21858. - VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
  21859. - /* Pageblock could have been isolated meanwhile */
  21860. - if (unlikely(isolated_pageblocks))
  21861. - mt = get_pageblock_migratetype(page);
  21862. -
  21863. - if (bulkfree_pcp_prepare(page))
  21864. - continue;
  21865. -
  21866. - __free_one_page(page, page_to_pfn(page), zone, 0, mt);
  21867. - trace_mm_page_pcpu_drain(page, 0, mt);
  21868. + list_add(&page->lru, dst);
  21869. } while (--count && --batch_free && !list_empty(list));
  21870. }
  21871. - spin_unlock(&zone->lock);
  21872. }
  21873. static void free_one_page(struct zone *zone,
  21874. @@ -1148,7 +1185,9 @@
  21875. int migratetype)
  21876. {
  21877. unsigned long nr_scanned;
  21878. - spin_lock(&zone->lock);
  21879. + unsigned long flags;
  21880. +
  21881. + spin_lock_irqsave(&zone->lock, flags);
  21882. nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
  21883. if (nr_scanned)
  21884. __mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned);
  21885. @@ -1158,7 +1197,7 @@
  21886. migratetype = get_pfnblock_migratetype(page, pfn);
  21887. }
  21888. __free_one_page(page, pfn, zone, order, migratetype);
  21889. - spin_unlock(&zone->lock);
  21890. + spin_unlock_irqrestore(&zone->lock, flags);
  21891. }
  21892. static void __meminit __init_single_page(struct page *page, unsigned long pfn,
  21893. @@ -1244,10 +1283,10 @@
  21894. return;
  21895. migratetype = get_pfnblock_migratetype(page, pfn);
  21896. - local_irq_save(flags);
  21897. + local_lock_irqsave(pa_lock, flags);
  21898. __count_vm_events(PGFREE, 1 << order);
  21899. free_one_page(page_zone(page), page, pfn, order, migratetype);
  21900. - local_irq_restore(flags);
  21901. + local_unlock_irqrestore(pa_lock, flags);
  21902. }
  21903. static void __init __free_pages_boot_core(struct page *page, unsigned int order)
  21904. @@ -2246,16 +2285,18 @@
  21905. void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
  21906. {
  21907. unsigned long flags;
  21908. + LIST_HEAD(dst);
  21909. int to_drain, batch;
  21910. - local_irq_save(flags);
  21911. + local_lock_irqsave(pa_lock, flags);
  21912. batch = READ_ONCE(pcp->batch);
  21913. to_drain = min(pcp->count, batch);
  21914. if (to_drain > 0) {
  21915. - free_pcppages_bulk(zone, to_drain, pcp);
  21916. + isolate_pcp_pages(to_drain, pcp, &dst);
  21917. pcp->count -= to_drain;
  21918. }
  21919. - local_irq_restore(flags);
  21920. + local_unlock_irqrestore(pa_lock, flags);
  21921. + free_pcppages_bulk(zone, to_drain, &dst);
  21922. }
  21923. #endif
  21924. @@ -2271,16 +2312,21 @@
  21925. unsigned long flags;
  21926. struct per_cpu_pageset *pset;
  21927. struct per_cpu_pages *pcp;
  21928. + LIST_HEAD(dst);
  21929. + int count;
  21930. - local_irq_save(flags);
  21931. + cpu_lock_irqsave(cpu, flags);
  21932. pset = per_cpu_ptr(zone->pageset, cpu);
  21933. pcp = &pset->pcp;
  21934. - if (pcp->count) {
  21935. - free_pcppages_bulk(zone, pcp->count, pcp);
  21936. + count = pcp->count;
  21937. + if (count) {
  21938. + isolate_pcp_pages(count, pcp, &dst);
  21939. pcp->count = 0;
  21940. }
  21941. - local_irq_restore(flags);
  21942. + cpu_unlock_irqrestore(cpu, flags);
  21943. + if (count)
  21944. + free_pcppages_bulk(zone, count, &dst);
  21945. }
  21946. /*
  21947. @@ -2366,8 +2412,17 @@
  21948. else
  21949. cpumask_clear_cpu(cpu, &cpus_with_pcps);
  21950. }
  21951. +#ifndef CONFIG_PREEMPT_RT_BASE
  21952. on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages,
  21953. zone, 1);
  21954. +#else
  21955. + for_each_cpu(cpu, &cpus_with_pcps) {
  21956. + if (zone)
  21957. + drain_pages_zone(cpu, zone);
  21958. + else
  21959. + drain_pages(cpu);
  21960. + }
  21961. +#endif
  21962. }
  21963. #ifdef CONFIG_HIBERNATION
  21964. @@ -2427,7 +2482,7 @@
  21965. migratetype = get_pfnblock_migratetype(page, pfn);
  21966. set_pcppage_migratetype(page, migratetype);
  21967. - local_irq_save(flags);
  21968. + local_lock_irqsave(pa_lock, flags);
  21969. __count_vm_event(PGFREE);
  21970. /*
  21971. @@ -2453,12 +2508,17 @@
  21972. pcp->count++;
  21973. if (pcp->count >= pcp->high) {
  21974. unsigned long batch = READ_ONCE(pcp->batch);
  21975. - free_pcppages_bulk(zone, batch, pcp);
  21976. + LIST_HEAD(dst);
  21977. +
  21978. + isolate_pcp_pages(batch, pcp, &dst);
  21979. pcp->count -= batch;
  21980. + local_unlock_irqrestore(pa_lock, flags);
  21981. + free_pcppages_bulk(zone, batch, &dst);
  21982. + return;
  21983. }
  21984. out:
  21985. - local_irq_restore(flags);
  21986. + local_unlock_irqrestore(pa_lock, flags);
  21987. }
  21988. /*
  21989. @@ -2600,7 +2660,7 @@
  21990. struct per_cpu_pages *pcp;
  21991. struct list_head *list;
  21992. - local_irq_save(flags);
  21993. + local_lock_irqsave(pa_lock, flags);
  21994. do {
  21995. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  21996. list = &pcp->lists[migratetype];
  21997. @@ -2627,7 +2687,7 @@
  21998. * allocate greater than order-1 page units with __GFP_NOFAIL.
  21999. */
  22000. WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
  22001. - spin_lock_irqsave(&zone->lock, flags);
  22002. + local_spin_lock_irqsave(pa_lock, &zone->lock, flags);
  22003. do {
  22004. page = NULL;
  22005. @@ -2639,22 +2699,24 @@
  22006. if (!page)
  22007. page = __rmqueue(zone, order, migratetype);
  22008. } while (page && check_new_pages(page, order));
  22009. - spin_unlock(&zone->lock);
  22010. - if (!page)
  22011. + if (!page) {
  22012. + spin_unlock(&zone->lock);
  22013. goto failed;
  22014. + }
  22015. __mod_zone_freepage_state(zone, -(1 << order),
  22016. get_pcppage_migratetype(page));
  22017. + spin_unlock(&zone->lock);
  22018. }
  22019. __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
  22020. zone_statistics(preferred_zone, zone, gfp_flags);
  22021. - local_irq_restore(flags);
  22022. + local_unlock_irqrestore(pa_lock, flags);
  22023. VM_BUG_ON_PAGE(bad_range(zone, page), page);
  22024. return page;
  22025. failed:
  22026. - local_irq_restore(flags);
  22027. + local_unlock_irqrestore(pa_lock, flags);
  22028. return NULL;
  22029. }
  22030. @@ -6531,7 +6593,9 @@
  22031. int cpu = (unsigned long)hcpu;
  22032. if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
  22033. + local_lock_irq_on(swapvec_lock, cpu);
  22034. lru_add_drain_cpu(cpu);
  22035. + local_unlock_irq_on(swapvec_lock, cpu);
  22036. drain_pages(cpu);
  22037. /*
  22038. @@ -6557,6 +6621,7 @@
  22039. void __init page_alloc_init(void)
  22040. {
  22041. hotcpu_notifier(page_alloc_cpu_notify, 0);
  22042. + local_irq_lock_init(pa_lock);
  22043. }
  22044. /*
  22045. @@ -7383,7 +7448,7 @@
  22046. struct per_cpu_pageset *pset;
  22047. /* avoid races with drain_pages() */
  22048. - local_irq_save(flags);
  22049. + local_lock_irqsave(pa_lock, flags);
  22050. if (zone->pageset != &boot_pageset) {
  22051. for_each_online_cpu(cpu) {
  22052. pset = per_cpu_ptr(zone->pageset, cpu);
  22053. @@ -7392,7 +7457,7 @@
  22054. free_percpu(zone->pageset);
  22055. zone->pageset = &boot_pageset;
  22056. }
  22057. - local_irq_restore(flags);
  22058. + local_unlock_irqrestore(pa_lock, flags);
  22059. }
  22060. #ifdef CONFIG_MEMORY_HOTREMOVE
  22061. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/percpu.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/percpu.c
  22062. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/percpu.c 2017-04-16 10:38:30.000000000 +0200
  22063. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/percpu.c 2017-04-18 17:54:26.000000000 +0200
  22064. @@ -1283,18 +1283,7 @@
  22065. }
  22066. EXPORT_SYMBOL_GPL(free_percpu);
  22067. -/**
  22068. - * is_kernel_percpu_address - test whether address is from static percpu area
  22069. - * @addr: address to test
  22070. - *
  22071. - * Test whether @addr belongs to in-kernel static percpu area. Module
  22072. - * static percpu areas are not considered. For those, use
  22073. - * is_module_percpu_address().
  22074. - *
  22075. - * RETURNS:
  22076. - * %true if @addr is from in-kernel static percpu area, %false otherwise.
  22077. - */
  22078. -bool is_kernel_percpu_address(unsigned long addr)
  22079. +bool __is_kernel_percpu_address(unsigned long addr, unsigned long *can_addr)
  22080. {
  22081. #ifdef CONFIG_SMP
  22082. const size_t static_size = __per_cpu_end - __per_cpu_start;
  22083. @@ -1303,16 +1292,39 @@
  22084. for_each_possible_cpu(cpu) {
  22085. void *start = per_cpu_ptr(base, cpu);
  22086. + void *va = (void *)addr;
  22087. - if ((void *)addr >= start && (void *)addr < start + static_size)
  22088. + if (va >= start && va < start + static_size) {
  22089. + if (can_addr) {
  22090. + *can_addr = (unsigned long) (va - start);
  22091. + *can_addr += (unsigned long)
  22092. + per_cpu_ptr(base, get_boot_cpu_id());
  22093. + }
  22094. return true;
  22095. - }
  22096. + }
  22097. + }
  22098. #endif
  22099. /* on UP, can't distinguish from other static vars, always false */
  22100. return false;
  22101. }
  22102. /**
  22103. + * is_kernel_percpu_address - test whether address is from static percpu area
  22104. + * @addr: address to test
  22105. + *
  22106. + * Test whether @addr belongs to in-kernel static percpu area. Module
  22107. + * static percpu areas are not considered. For those, use
  22108. + * is_module_percpu_address().
  22109. + *
  22110. + * RETURNS:
  22111. + * %true if @addr is from in-kernel static percpu area, %false otherwise.
  22112. + */
  22113. +bool is_kernel_percpu_address(unsigned long addr)
  22114. +{
  22115. + return __is_kernel_percpu_address(addr, NULL);
  22116. +}
  22117. +
  22118. +/**
  22119. * per_cpu_ptr_to_phys - convert translated percpu address to physical address
  22120. * @addr: the address to be converted to physical address
  22121. *
  22122. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/slab.h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/slab.h
  22123. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/slab.h 2017-04-16 10:38:30.000000000 +0200
  22124. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/slab.h 2017-04-18 17:54:26.000000000 +0200
  22125. @@ -426,7 +426,11 @@
  22126. * The slab lists for all objects.
  22127. */
  22128. struct kmem_cache_node {
  22129. +#ifdef CONFIG_SLUB
  22130. + raw_spinlock_t list_lock;
  22131. +#else
  22132. spinlock_t list_lock;
  22133. +#endif
  22134. #ifdef CONFIG_SLAB
  22135. struct list_head slabs_partial; /* partial list first, better asm code */
  22136. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/slub.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/slub.c
  22137. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/slub.c 2017-04-16 10:38:30.000000000 +0200
  22138. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/slub.c 2017-04-18 17:54:26.000000000 +0200
  22139. @@ -1141,7 +1141,7 @@
  22140. unsigned long uninitialized_var(flags);
  22141. int ret = 0;
  22142. - spin_lock_irqsave(&n->list_lock, flags);
  22143. + raw_spin_lock_irqsave(&n->list_lock, flags);
  22144. slab_lock(page);
  22145. if (s->flags & SLAB_CONSISTENCY_CHECKS) {
  22146. @@ -1176,7 +1176,7 @@
  22147. bulk_cnt, cnt);
  22148. slab_unlock(page);
  22149. - spin_unlock_irqrestore(&n->list_lock, flags);
  22150. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  22151. if (!ret)
  22152. slab_fix(s, "Object at 0x%p not freed", object);
  22153. return ret;
  22154. @@ -1304,6 +1304,12 @@
  22155. #endif /* CONFIG_SLUB_DEBUG */
  22156. +struct slub_free_list {
  22157. + raw_spinlock_t lock;
  22158. + struct list_head list;
  22159. +};
  22160. +static DEFINE_PER_CPU(struct slub_free_list, slub_free_list);
  22161. +
  22162. /*
  22163. * Hooks for other subsystems that check memory allocations. In a typical
  22164. * production configuration these hooks all should produce no code at all.
  22165. @@ -1527,10 +1533,17 @@
  22166. void *start, *p;
  22167. int idx, order;
  22168. bool shuffle;
  22169. + bool enableirqs = false;
  22170. flags &= gfp_allowed_mask;
  22171. if (gfpflags_allow_blocking(flags))
  22172. + enableirqs = true;
  22173. +#ifdef CONFIG_PREEMPT_RT_FULL
  22174. + if (system_state == SYSTEM_RUNNING)
  22175. + enableirqs = true;
  22176. +#endif
  22177. + if (enableirqs)
  22178. local_irq_enable();
  22179. flags |= s->allocflags;
  22180. @@ -1605,7 +1618,7 @@
  22181. page->frozen = 1;
  22182. out:
  22183. - if (gfpflags_allow_blocking(flags))
  22184. + if (enableirqs)
  22185. local_irq_disable();
  22186. if (!page)
  22187. return NULL;
  22188. @@ -1664,6 +1677,16 @@
  22189. __free_pages(page, order);
  22190. }
  22191. +static void free_delayed(struct list_head *h)
  22192. +{
  22193. + while(!list_empty(h)) {
  22194. + struct page *page = list_first_entry(h, struct page, lru);
  22195. +
  22196. + list_del(&page->lru);
  22197. + __free_slab(page->slab_cache, page);
  22198. + }
  22199. +}
  22200. +
  22201. #define need_reserve_slab_rcu \
  22202. (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))
  22203. @@ -1695,6 +1718,12 @@
  22204. }
  22205. call_rcu(head, rcu_free_slab);
  22206. + } else if (irqs_disabled()) {
  22207. + struct slub_free_list *f = this_cpu_ptr(&slub_free_list);
  22208. +
  22209. + raw_spin_lock(&f->lock);
  22210. + list_add(&page->lru, &f->list);
  22211. + raw_spin_unlock(&f->lock);
  22212. } else
  22213. __free_slab(s, page);
  22214. }
  22215. @@ -1802,7 +1831,7 @@
  22216. if (!n || !n->nr_partial)
  22217. return NULL;
  22218. - spin_lock(&n->list_lock);
  22219. + raw_spin_lock(&n->list_lock);
  22220. list_for_each_entry_safe(page, page2, &n->partial, lru) {
  22221. void *t;
  22222. @@ -1827,7 +1856,7 @@
  22223. break;
  22224. }
  22225. - spin_unlock(&n->list_lock);
  22226. + raw_spin_unlock(&n->list_lock);
  22227. return object;
  22228. }
  22229. @@ -2073,7 +2102,7 @@
  22230. * that acquire_slab() will see a slab page that
  22231. * is frozen
  22232. */
  22233. - spin_lock(&n->list_lock);
  22234. + raw_spin_lock(&n->list_lock);
  22235. }
  22236. } else {
  22237. m = M_FULL;
  22238. @@ -2084,7 +2113,7 @@
  22239. * slabs from diagnostic functions will not see
  22240. * any frozen slabs.
  22241. */
  22242. - spin_lock(&n->list_lock);
  22243. + raw_spin_lock(&n->list_lock);
  22244. }
  22245. }
  22246. @@ -2119,7 +2148,7 @@
  22247. goto redo;
  22248. if (lock)
  22249. - spin_unlock(&n->list_lock);
  22250. + raw_spin_unlock(&n->list_lock);
  22251. if (m == M_FREE) {
  22252. stat(s, DEACTIVATE_EMPTY);
  22253. @@ -2151,10 +2180,10 @@
  22254. n2 = get_node(s, page_to_nid(page));
  22255. if (n != n2) {
  22256. if (n)
  22257. - spin_unlock(&n->list_lock);
  22258. + raw_spin_unlock(&n->list_lock);
  22259. n = n2;
  22260. - spin_lock(&n->list_lock);
  22261. + raw_spin_lock(&n->list_lock);
  22262. }
  22263. do {
  22264. @@ -2183,7 +2212,7 @@
  22265. }
  22266. if (n)
  22267. - spin_unlock(&n->list_lock);
  22268. + raw_spin_unlock(&n->list_lock);
  22269. while (discard_page) {
  22270. page = discard_page;
  22271. @@ -2222,14 +2251,21 @@
  22272. pobjects = oldpage->pobjects;
  22273. pages = oldpage->pages;
  22274. if (drain && pobjects > s->cpu_partial) {
  22275. + struct slub_free_list *f;
  22276. unsigned long flags;
  22277. + LIST_HEAD(tofree);
  22278. /*
  22279. * partial array is full. Move the existing
  22280. * set to the per node partial list.
  22281. */
  22282. local_irq_save(flags);
  22283. unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
  22284. + f = this_cpu_ptr(&slub_free_list);
  22285. + raw_spin_lock(&f->lock);
  22286. + list_splice_init(&f->list, &tofree);
  22287. + raw_spin_unlock(&f->lock);
  22288. local_irq_restore(flags);
  22289. + free_delayed(&tofree);
  22290. oldpage = NULL;
  22291. pobjects = 0;
  22292. pages = 0;
  22293. @@ -2301,7 +2337,22 @@
  22294. static void flush_all(struct kmem_cache *s)
  22295. {
  22296. + LIST_HEAD(tofree);
  22297. + int cpu;
  22298. +
  22299. on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
  22300. + for_each_online_cpu(cpu) {
  22301. + struct slub_free_list *f;
  22302. +
  22303. + if (!has_cpu_slab(cpu, s))
  22304. + continue;
  22305. +
  22306. + f = &per_cpu(slub_free_list, cpu);
  22307. + raw_spin_lock_irq(&f->lock);
  22308. + list_splice_init(&f->list, &tofree);
  22309. + raw_spin_unlock_irq(&f->lock);
  22310. + free_delayed(&tofree);
  22311. + }
  22312. }
  22313. /*
  22314. @@ -2356,10 +2407,10 @@
  22315. unsigned long x = 0;
  22316. struct page *page;
  22317. - spin_lock_irqsave(&n->list_lock, flags);
  22318. + raw_spin_lock_irqsave(&n->list_lock, flags);
  22319. list_for_each_entry(page, &n->partial, lru)
  22320. x += get_count(page);
  22321. - spin_unlock_irqrestore(&n->list_lock, flags);
  22322. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  22323. return x;
  22324. }
  22325. #endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
  22326. @@ -2497,8 +2548,10 @@
  22327. * already disabled (which is the case for bulk allocation).
  22328. */
  22329. static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
  22330. - unsigned long addr, struct kmem_cache_cpu *c)
  22331. + unsigned long addr, struct kmem_cache_cpu *c,
  22332. + struct list_head *to_free)
  22333. {
  22334. + struct slub_free_list *f;
  22335. void *freelist;
  22336. struct page *page;
  22337. @@ -2558,6 +2611,13 @@
  22338. VM_BUG_ON(!c->page->frozen);
  22339. c->freelist = get_freepointer(s, freelist);
  22340. c->tid = next_tid(c->tid);
  22341. +
  22342. +out:
  22343. + f = this_cpu_ptr(&slub_free_list);
  22344. + raw_spin_lock(&f->lock);
  22345. + list_splice_init(&f->list, to_free);
  22346. + raw_spin_unlock(&f->lock);
  22347. +
  22348. return freelist;
  22349. new_slab:
  22350. @@ -2589,7 +2649,7 @@
  22351. deactivate_slab(s, page, get_freepointer(s, freelist));
  22352. c->page = NULL;
  22353. c->freelist = NULL;
  22354. - return freelist;
  22355. + goto out;
  22356. }
  22357. /*
  22358. @@ -2601,6 +2661,7 @@
  22359. {
  22360. void *p;
  22361. unsigned long flags;
  22362. + LIST_HEAD(tofree);
  22363. local_irq_save(flags);
  22364. #ifdef CONFIG_PREEMPT
  22365. @@ -2612,8 +2673,9 @@
  22366. c = this_cpu_ptr(s->cpu_slab);
  22367. #endif
  22368. - p = ___slab_alloc(s, gfpflags, node, addr, c);
  22369. + p = ___slab_alloc(s, gfpflags, node, addr, c, &tofree);
  22370. local_irq_restore(flags);
  22371. + free_delayed(&tofree);
  22372. return p;
  22373. }
  22374. @@ -2799,7 +2861,7 @@
  22375. do {
  22376. if (unlikely(n)) {
  22377. - spin_unlock_irqrestore(&n->list_lock, flags);
  22378. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  22379. n = NULL;
  22380. }
  22381. prior = page->freelist;
  22382. @@ -2831,7 +2893,7 @@
  22383. * Otherwise the list_lock will synchronize with
  22384. * other processors updating the list of slabs.
  22385. */
  22386. - spin_lock_irqsave(&n->list_lock, flags);
  22387. + raw_spin_lock_irqsave(&n->list_lock, flags);
  22388. }
  22389. }
  22390. @@ -2873,7 +2935,7 @@
  22391. add_partial(n, page, DEACTIVATE_TO_TAIL);
  22392. stat(s, FREE_ADD_PARTIAL);
  22393. }
  22394. - spin_unlock_irqrestore(&n->list_lock, flags);
  22395. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  22396. return;
  22397. slab_empty:
  22398. @@ -2888,7 +2950,7 @@
  22399. remove_full(s, n, page);
  22400. }
  22401. - spin_unlock_irqrestore(&n->list_lock, flags);
  22402. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  22403. stat(s, FREE_SLAB);
  22404. discard_slab(s, page);
  22405. }
  22406. @@ -3093,6 +3155,7 @@
  22407. void **p)
  22408. {
  22409. struct kmem_cache_cpu *c;
  22410. + LIST_HEAD(to_free);
  22411. int i;
  22412. /* memcg and kmem_cache debug support */
  22413. @@ -3116,7 +3179,7 @@
  22414. * of re-populating per CPU c->freelist
  22415. */
  22416. p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
  22417. - _RET_IP_, c);
  22418. + _RET_IP_, c, &to_free);
  22419. if (unlikely(!p[i]))
  22420. goto error;
  22421. @@ -3128,6 +3191,7 @@
  22422. }
  22423. c->tid = next_tid(c->tid);
  22424. local_irq_enable();
  22425. + free_delayed(&to_free);
  22426. /* Clear memory outside IRQ disabled fastpath loop */
  22427. if (unlikely(flags & __GFP_ZERO)) {
  22428. @@ -3275,7 +3339,7 @@
  22429. init_kmem_cache_node(struct kmem_cache_node *n)
  22430. {
  22431. n->nr_partial = 0;
  22432. - spin_lock_init(&n->list_lock);
  22433. + raw_spin_lock_init(&n->list_lock);
  22434. INIT_LIST_HEAD(&n->partial);
  22435. #ifdef CONFIG_SLUB_DEBUG
  22436. atomic_long_set(&n->nr_slabs, 0);
  22437. @@ -3619,6 +3683,10 @@
  22438. const char *text)
  22439. {
  22440. #ifdef CONFIG_SLUB_DEBUG
  22441. +#ifdef CONFIG_PREEMPT_RT_BASE
  22442. + /* XXX move out of irq-off section */
  22443. + slab_err(s, page, text, s->name);
  22444. +#else
  22445. void *addr = page_address(page);
  22446. void *p;
  22447. unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) *
  22448. @@ -3639,6 +3707,7 @@
  22449. slab_unlock(page);
  22450. kfree(map);
  22451. #endif
  22452. +#endif
  22453. }
  22454. /*
  22455. @@ -3652,7 +3721,7 @@
  22456. struct page *page, *h;
  22457. BUG_ON(irqs_disabled());
  22458. - spin_lock_irq(&n->list_lock);
  22459. + raw_spin_lock_irq(&n->list_lock);
  22460. list_for_each_entry_safe(page, h, &n->partial, lru) {
  22461. if (!page->inuse) {
  22462. remove_partial(n, page);
  22463. @@ -3662,7 +3731,7 @@
  22464. "Objects remaining in %s on __kmem_cache_shutdown()");
  22465. }
  22466. }
  22467. - spin_unlock_irq(&n->list_lock);
  22468. + raw_spin_unlock_irq(&n->list_lock);
  22469. list_for_each_entry_safe(page, h, &discard, lru)
  22470. discard_slab(s, page);
  22471. @@ -3905,7 +3974,7 @@
  22472. for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
  22473. INIT_LIST_HEAD(promote + i);
  22474. - spin_lock_irqsave(&n->list_lock, flags);
  22475. + raw_spin_lock_irqsave(&n->list_lock, flags);
  22476. /*
  22477. * Build lists of slabs to discard or promote.
  22478. @@ -3936,7 +4005,7 @@
  22479. for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
  22480. list_splice(promote + i, &n->partial);
  22481. - spin_unlock_irqrestore(&n->list_lock, flags);
  22482. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  22483. /* Release empty slabs */
  22484. list_for_each_entry_safe(page, t, &discard, lru)
  22485. @@ -4112,6 +4181,12 @@
  22486. {
  22487. static __initdata struct kmem_cache boot_kmem_cache,
  22488. boot_kmem_cache_node;
  22489. + int cpu;
  22490. +
  22491. + for_each_possible_cpu(cpu) {
  22492. + raw_spin_lock_init(&per_cpu(slub_free_list, cpu).lock);
  22493. + INIT_LIST_HEAD(&per_cpu(slub_free_list, cpu).list);
  22494. + }
  22495. if (debug_guardpage_minorder())
  22496. slub_max_order = 0;
  22497. @@ -4320,7 +4395,7 @@
  22498. struct page *page;
  22499. unsigned long flags;
  22500. - spin_lock_irqsave(&n->list_lock, flags);
  22501. + raw_spin_lock_irqsave(&n->list_lock, flags);
  22502. list_for_each_entry(page, &n->partial, lru) {
  22503. validate_slab_slab(s, page, map);
  22504. @@ -4342,7 +4417,7 @@
  22505. s->name, count, atomic_long_read(&n->nr_slabs));
  22506. out:
  22507. - spin_unlock_irqrestore(&n->list_lock, flags);
  22508. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  22509. return count;
  22510. }
  22511. @@ -4530,12 +4605,12 @@
  22512. if (!atomic_long_read(&n->nr_slabs))
  22513. continue;
  22514. - spin_lock_irqsave(&n->list_lock, flags);
  22515. + raw_spin_lock_irqsave(&n->list_lock, flags);
  22516. list_for_each_entry(page, &n->partial, lru)
  22517. process_slab(&t, s, page, alloc, map);
  22518. list_for_each_entry(page, &n->full, lru)
  22519. process_slab(&t, s, page, alloc, map);
  22520. - spin_unlock_irqrestore(&n->list_lock, flags);
  22521. + raw_spin_unlock_irqrestore(&n->list_lock, flags);
  22522. }
  22523. for (i = 0; i < t.count; i++) {
  22524. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/swap.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/swap.c
  22525. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/swap.c 2017-04-16 10:38:30.000000000 +0200
  22526. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/swap.c 2017-04-18 17:54:26.000000000 +0200
  22527. @@ -32,6 +32,7 @@
  22528. #include <linux/memcontrol.h>
  22529. #include <linux/gfp.h>
  22530. #include <linux/uio.h>
  22531. +#include <linux/locallock.h>
  22532. #include <linux/hugetlb.h>
  22533. #include <linux/page_idle.h>
  22534. @@ -50,6 +51,8 @@
  22535. #ifdef CONFIG_SMP
  22536. static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
  22537. #endif
  22538. +static DEFINE_LOCAL_IRQ_LOCK(rotate_lock);
  22539. +DEFINE_LOCAL_IRQ_LOCK(swapvec_lock);
  22540. /*
  22541. * This path almost never happens for VM activity - pages are normally
  22542. @@ -240,11 +243,11 @@
  22543. unsigned long flags;
  22544. get_page(page);
  22545. - local_irq_save(flags);
  22546. + local_lock_irqsave(rotate_lock, flags);
  22547. pvec = this_cpu_ptr(&lru_rotate_pvecs);
  22548. if (!pagevec_add(pvec, page) || PageCompound(page))
  22549. pagevec_move_tail(pvec);
  22550. - local_irq_restore(flags);
  22551. + local_unlock_irqrestore(rotate_lock, flags);
  22552. }
  22553. }
  22554. @@ -294,12 +297,13 @@
  22555. {
  22556. page = compound_head(page);
  22557. if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
  22558. - struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);
  22559. + struct pagevec *pvec = &get_locked_var(swapvec_lock,
  22560. + activate_page_pvecs);
  22561. get_page(page);
  22562. if (!pagevec_add(pvec, page) || PageCompound(page))
  22563. pagevec_lru_move_fn(pvec, __activate_page, NULL);
  22564. - put_cpu_var(activate_page_pvecs);
  22565. + put_locked_var(swapvec_lock, activate_page_pvecs);
  22566. }
  22567. }
  22568. @@ -326,7 +330,7 @@
  22569. static void __lru_cache_activate_page(struct page *page)
  22570. {
  22571. - struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
  22572. + struct pagevec *pvec = &get_locked_var(swapvec_lock, lru_add_pvec);
  22573. int i;
  22574. /*
  22575. @@ -348,7 +352,7 @@
  22576. }
  22577. }
  22578. - put_cpu_var(lru_add_pvec);
  22579. + put_locked_var(swapvec_lock, lru_add_pvec);
  22580. }
  22581. /*
  22582. @@ -390,12 +394,12 @@
  22583. static void __lru_cache_add(struct page *page)
  22584. {
  22585. - struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
  22586. + struct pagevec *pvec = &get_locked_var(swapvec_lock, lru_add_pvec);
  22587. get_page(page);
  22588. if (!pagevec_add(pvec, page) || PageCompound(page))
  22589. __pagevec_lru_add(pvec);
  22590. - put_cpu_var(lru_add_pvec);
  22591. + put_locked_var(swapvec_lock, lru_add_pvec);
  22592. }
  22593. /**
  22594. @@ -593,9 +597,15 @@
  22595. unsigned long flags;
  22596. /* No harm done if a racing interrupt already did this */
  22597. - local_irq_save(flags);
  22598. +#ifdef CONFIG_PREEMPT_RT_BASE
  22599. + local_lock_irqsave_on(rotate_lock, flags, cpu);
  22600. pagevec_move_tail(pvec);
  22601. - local_irq_restore(flags);
  22602. + local_unlock_irqrestore_on(rotate_lock, flags, cpu);
  22603. +#else
  22604. + local_lock_irqsave(rotate_lock, flags);
  22605. + pagevec_move_tail(pvec);
  22606. + local_unlock_irqrestore(rotate_lock, flags);
  22607. +#endif
  22608. }
  22609. pvec = &per_cpu(lru_deactivate_file_pvecs, cpu);
  22610. @@ -627,11 +637,12 @@
  22611. return;
  22612. if (likely(get_page_unless_zero(page))) {
  22613. - struct pagevec *pvec = &get_cpu_var(lru_deactivate_file_pvecs);
  22614. + struct pagevec *pvec = &get_locked_var(swapvec_lock,
  22615. + lru_deactivate_file_pvecs);
  22616. if (!pagevec_add(pvec, page) || PageCompound(page))
  22617. pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
  22618. - put_cpu_var(lru_deactivate_file_pvecs);
  22619. + put_locked_var(swapvec_lock, lru_deactivate_file_pvecs);
  22620. }
  22621. }
  22622. @@ -646,27 +657,31 @@
  22623. void deactivate_page(struct page *page)
  22624. {
  22625. if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
  22626. - struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
  22627. + struct pagevec *pvec = &get_locked_var(swapvec_lock,
  22628. + lru_deactivate_pvecs);
  22629. get_page(page);
  22630. if (!pagevec_add(pvec, page) || PageCompound(page))
  22631. pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
  22632. - put_cpu_var(lru_deactivate_pvecs);
  22633. + put_locked_var(swapvec_lock, lru_deactivate_pvecs);
  22634. }
  22635. }
  22636. void lru_add_drain(void)
  22637. {
  22638. - lru_add_drain_cpu(get_cpu());
  22639. - put_cpu();
  22640. + lru_add_drain_cpu(local_lock_cpu(swapvec_lock));
  22641. + local_unlock_cpu(swapvec_lock);
  22642. }
  22643. -static void lru_add_drain_per_cpu(struct work_struct *dummy)
  22644. +#ifdef CONFIG_PREEMPT_RT_BASE
  22645. +static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work)
  22646. {
  22647. - lru_add_drain();
  22648. + local_lock_on(swapvec_lock, cpu);
  22649. + lru_add_drain_cpu(cpu);
  22650. + local_unlock_on(swapvec_lock, cpu);
  22651. }
  22652. -static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
  22653. +#else
  22654. /*
  22655. * lru_add_drain_wq is used to do lru_add_drain_all() from a WQ_MEM_RECLAIM
  22656. @@ -686,6 +701,22 @@
  22657. }
  22658. early_initcall(lru_init);
  22659. +static void lru_add_drain_per_cpu(struct work_struct *dummy)
  22660. +{
  22661. + lru_add_drain();
  22662. +}
  22663. +
  22664. +static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
  22665. +static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work)
  22666. +{
  22667. + struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
  22668. +
  22669. + INIT_WORK(work, lru_add_drain_per_cpu);
  22670. + queue_work_on(cpu, lru_add_drain_wq, work);
  22671. + cpumask_set_cpu(cpu, has_work);
  22672. +}
  22673. +#endif
  22674. +
  22675. void lru_add_drain_all(void)
  22676. {
  22677. static DEFINE_MUTEX(lock);
  22678. @@ -697,21 +728,18 @@
  22679. cpumask_clear(&has_work);
  22680. for_each_online_cpu(cpu) {
  22681. - struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
  22682. -
  22683. if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) ||
  22684. pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
  22685. pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) ||
  22686. pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) ||
  22687. - need_activate_page_drain(cpu)) {
  22688. - INIT_WORK(work, lru_add_drain_per_cpu);
  22689. - queue_work_on(cpu, lru_add_drain_wq, work);
  22690. - cpumask_set_cpu(cpu, &has_work);
  22691. - }
  22692. + need_activate_page_drain(cpu))
  22693. + remote_lru_add_drain(cpu, &has_work);
  22694. }
  22695. +#ifndef CONFIG_PREEMPT_RT_BASE
  22696. for_each_cpu(cpu, &has_work)
  22697. flush_work(&per_cpu(lru_add_drain_work, cpu));
  22698. +#endif
  22699. put_online_cpus();
  22700. mutex_unlock(&lock);
  22701. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/truncate.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/truncate.c
  22702. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/truncate.c 2017-04-16 10:38:30.000000000 +0200
  22703. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/truncate.c 2017-04-18 17:54:26.000000000 +0200
  22704. @@ -62,9 +62,12 @@
  22705. * protected by mapping->tree_lock.
  22706. */
  22707. if (!workingset_node_shadows(node) &&
  22708. - !list_empty(&node->private_list))
  22709. - list_lru_del(&workingset_shadow_nodes,
  22710. + !list_empty(&node->private_list)) {
  22711. + local_lock(workingset_shadow_lock);
  22712. + list_lru_del(&__workingset_shadow_nodes,
  22713. &node->private_list);
  22714. + local_unlock(workingset_shadow_lock);
  22715. + }
  22716. __radix_tree_delete_node(&mapping->page_tree, node);
  22717. unlock:
  22718. spin_unlock_irq(&mapping->tree_lock);
  22719. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/vmalloc.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/vmalloc.c
  22720. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/vmalloc.c 2017-04-16 10:38:30.000000000 +0200
  22721. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/vmalloc.c 2017-04-18 17:54:26.000000000 +0200
  22722. @@ -845,7 +845,7 @@
  22723. struct vmap_block *vb;
  22724. struct vmap_area *va;
  22725. unsigned long vb_idx;
  22726. - int node, err;
  22727. + int node, err, cpu;
  22728. void *vaddr;
  22729. node = numa_node_id();
  22730. @@ -888,11 +888,12 @@
  22731. BUG_ON(err);
  22732. radix_tree_preload_end();
  22733. - vbq = &get_cpu_var(vmap_block_queue);
  22734. + cpu = get_cpu_light();
  22735. + vbq = this_cpu_ptr(&vmap_block_queue);
  22736. spin_lock(&vbq->lock);
  22737. list_add_tail_rcu(&vb->free_list, &vbq->free);
  22738. spin_unlock(&vbq->lock);
  22739. - put_cpu_var(vmap_block_queue);
  22740. + put_cpu_light();
  22741. return vaddr;
  22742. }
  22743. @@ -961,6 +962,7 @@
  22744. struct vmap_block *vb;
  22745. void *vaddr = NULL;
  22746. unsigned int order;
  22747. + int cpu;
  22748. BUG_ON(offset_in_page(size));
  22749. BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
  22750. @@ -975,7 +977,8 @@
  22751. order = get_order(size);
  22752. rcu_read_lock();
  22753. - vbq = &get_cpu_var(vmap_block_queue);
  22754. + cpu = get_cpu_light();
  22755. + vbq = this_cpu_ptr(&vmap_block_queue);
  22756. list_for_each_entry_rcu(vb, &vbq->free, free_list) {
  22757. unsigned long pages_off;
  22758. @@ -998,7 +1001,7 @@
  22759. break;
  22760. }
  22761. - put_cpu_var(vmap_block_queue);
  22762. + put_cpu_light();
  22763. rcu_read_unlock();
  22764. /* Allocate new block if nothing was found */
  22765. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/vmstat.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/vmstat.c
  22766. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/vmstat.c 2017-04-16 10:38:30.000000000 +0200
  22767. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/vmstat.c 2017-04-18 17:54:26.000000000 +0200
  22768. @@ -245,6 +245,7 @@
  22769. long x;
  22770. long t;
  22771. + preempt_disable_rt();
  22772. x = delta + __this_cpu_read(*p);
  22773. t = __this_cpu_read(pcp->stat_threshold);
  22774. @@ -254,6 +255,7 @@
  22775. x = 0;
  22776. }
  22777. __this_cpu_write(*p, x);
  22778. + preempt_enable_rt();
  22779. }
  22780. EXPORT_SYMBOL(__mod_zone_page_state);
  22781. @@ -265,6 +267,7 @@
  22782. long x;
  22783. long t;
  22784. + preempt_disable_rt();
  22785. x = delta + __this_cpu_read(*p);
  22786. t = __this_cpu_read(pcp->stat_threshold);
  22787. @@ -274,6 +277,7 @@
  22788. x = 0;
  22789. }
  22790. __this_cpu_write(*p, x);
  22791. + preempt_enable_rt();
  22792. }
  22793. EXPORT_SYMBOL(__mod_node_page_state);
  22794. @@ -306,6 +310,7 @@
  22795. s8 __percpu *p = pcp->vm_stat_diff + item;
  22796. s8 v, t;
  22797. + preempt_disable_rt();
  22798. v = __this_cpu_inc_return(*p);
  22799. t = __this_cpu_read(pcp->stat_threshold);
  22800. if (unlikely(v > t)) {
  22801. @@ -314,6 +319,7 @@
  22802. zone_page_state_add(v + overstep, zone, item);
  22803. __this_cpu_write(*p, -overstep);
  22804. }
  22805. + preempt_enable_rt();
  22806. }
  22807. void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
  22808. @@ -322,6 +328,7 @@
  22809. s8 __percpu *p = pcp->vm_node_stat_diff + item;
  22810. s8 v, t;
  22811. + preempt_disable_rt();
  22812. v = __this_cpu_inc_return(*p);
  22813. t = __this_cpu_read(pcp->stat_threshold);
  22814. if (unlikely(v > t)) {
  22815. @@ -330,6 +337,7 @@
  22816. node_page_state_add(v + overstep, pgdat, item);
  22817. __this_cpu_write(*p, -overstep);
  22818. }
  22819. + preempt_enable_rt();
  22820. }
  22821. void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
  22822. @@ -350,6 +358,7 @@
  22823. s8 __percpu *p = pcp->vm_stat_diff + item;
  22824. s8 v, t;
  22825. + preempt_disable_rt();
  22826. v = __this_cpu_dec_return(*p);
  22827. t = __this_cpu_read(pcp->stat_threshold);
  22828. if (unlikely(v < - t)) {
  22829. @@ -358,6 +367,7 @@
  22830. zone_page_state_add(v - overstep, zone, item);
  22831. __this_cpu_write(*p, overstep);
  22832. }
  22833. + preempt_enable_rt();
  22834. }
  22835. void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
  22836. @@ -366,6 +376,7 @@
  22837. s8 __percpu *p = pcp->vm_node_stat_diff + item;
  22838. s8 v, t;
  22839. + preempt_disable_rt();
  22840. v = __this_cpu_dec_return(*p);
  22841. t = __this_cpu_read(pcp->stat_threshold);
  22842. if (unlikely(v < - t)) {
  22843. @@ -374,6 +385,7 @@
  22844. node_page_state_add(v - overstep, pgdat, item);
  22845. __this_cpu_write(*p, overstep);
  22846. }
  22847. + preempt_enable_rt();
  22848. }
  22849. void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
  22850. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/workingset.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/workingset.c
  22851. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/workingset.c 2017-04-16 10:38:30.000000000 +0200
  22852. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/workingset.c 2017-04-18 17:54:26.000000000 +0200
  22853. @@ -334,7 +334,8 @@
  22854. * point where they would still be useful.
  22855. */
  22856. -struct list_lru workingset_shadow_nodes;
  22857. +struct list_lru __workingset_shadow_nodes;
  22858. +DEFINE_LOCAL_IRQ_LOCK(workingset_shadow_lock);
  22859. static unsigned long count_shadow_nodes(struct shrinker *shrinker,
  22860. struct shrink_control *sc)
  22861. @@ -344,9 +345,9 @@
  22862. unsigned long pages;
  22863. /* list_lru lock nests inside IRQ-safe mapping->tree_lock */
  22864. - local_irq_disable();
  22865. - shadow_nodes = list_lru_shrink_count(&workingset_shadow_nodes, sc);
  22866. - local_irq_enable();
  22867. + local_lock_irq(workingset_shadow_lock);
  22868. + shadow_nodes = list_lru_shrink_count(&__workingset_shadow_nodes, sc);
  22869. + local_unlock_irq(workingset_shadow_lock);
  22870. if (sc->memcg) {
  22871. pages = mem_cgroup_node_nr_lru_pages(sc->memcg, sc->nid,
  22872. @@ -438,9 +439,9 @@
  22873. spin_unlock(&mapping->tree_lock);
  22874. ret = LRU_REMOVED_RETRY;
  22875. out:
  22876. - local_irq_enable();
  22877. + local_unlock_irq(workingset_shadow_lock);
  22878. cond_resched();
  22879. - local_irq_disable();
  22880. + local_lock_irq(workingset_shadow_lock);
  22881. spin_lock(lru_lock);
  22882. return ret;
  22883. }
  22884. @@ -451,10 +452,10 @@
  22885. unsigned long ret;
  22886. /* list_lru lock nests inside IRQ-safe mapping->tree_lock */
  22887. - local_irq_disable();
  22888. - ret = list_lru_shrink_walk(&workingset_shadow_nodes, sc,
  22889. + local_lock_irq(workingset_shadow_lock);
  22890. + ret = list_lru_shrink_walk(&__workingset_shadow_nodes, sc,
  22891. shadow_lru_isolate, NULL);
  22892. - local_irq_enable();
  22893. + local_unlock_irq(workingset_shadow_lock);
  22894. return ret;
  22895. }
  22896. @@ -492,7 +493,7 @@
  22897. pr_info("workingset: timestamp_bits=%d max_order=%d bucket_order=%u\n",
  22898. timestamp_bits, max_order, bucket_order);
  22899. - ret = __list_lru_init(&workingset_shadow_nodes, true, &shadow_nodes_key);
  22900. + ret = __list_lru_init(&__workingset_shadow_nodes, true, &shadow_nodes_key);
  22901. if (ret)
  22902. goto err;
  22903. ret = register_shrinker(&workingset_shadow_shrinker);
  22904. @@ -500,7 +501,7 @@
  22905. goto err_list_lru;
  22906. return 0;
  22907. err_list_lru:
  22908. - list_lru_destroy(&workingset_shadow_nodes);
  22909. + list_lru_destroy(&__workingset_shadow_nodes);
  22910. err:
  22911. return ret;
  22912. }
  22913. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/zsmalloc.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/zsmalloc.c
  22914. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/mm/zsmalloc.c 2017-04-16 10:38:30.000000000 +0200
  22915. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/mm/zsmalloc.c 2017-04-18 17:54:26.000000000 +0200
  22916. @@ -53,6 +53,7 @@
  22917. #include <linux/mount.h>
  22918. #include <linux/migrate.h>
  22919. #include <linux/pagemap.h>
  22920. +#include <linux/locallock.h>
  22921. #define ZSPAGE_MAGIC 0x58
  22922. @@ -70,9 +71,22 @@
  22923. */
  22924. #define ZS_MAX_ZSPAGE_ORDER 2
  22925. #define ZS_MAX_PAGES_PER_ZSPAGE (_AC(1, UL) << ZS_MAX_ZSPAGE_ORDER)
  22926. -
  22927. #define ZS_HANDLE_SIZE (sizeof(unsigned long))
  22928. +#ifdef CONFIG_PREEMPT_RT_FULL
  22929. +
  22930. +struct zsmalloc_handle {
  22931. + unsigned long addr;
  22932. + struct mutex lock;
  22933. +};
  22934. +
  22935. +#define ZS_HANDLE_ALLOC_SIZE (sizeof(struct zsmalloc_handle))
  22936. +
  22937. +#else
  22938. +
  22939. +#define ZS_HANDLE_ALLOC_SIZE (sizeof(unsigned long))
  22940. +#endif
  22941. +
  22942. /*
  22943. * Object location (<PFN>, <obj_idx>) is encoded as
  22944. * as single (unsigned long) handle value.
  22945. @@ -327,7 +341,7 @@
  22946. static int create_cache(struct zs_pool *pool)
  22947. {
  22948. - pool->handle_cachep = kmem_cache_create("zs_handle", ZS_HANDLE_SIZE,
  22949. + pool->handle_cachep = kmem_cache_create("zs_handle", ZS_HANDLE_ALLOC_SIZE,
  22950. 0, 0, NULL);
  22951. if (!pool->handle_cachep)
  22952. return 1;
  22953. @@ -351,10 +365,27 @@
  22954. static unsigned long cache_alloc_handle(struct zs_pool *pool, gfp_t gfp)
  22955. {
  22956. - return (unsigned long)kmem_cache_alloc(pool->handle_cachep,
  22957. - gfp & ~(__GFP_HIGHMEM|__GFP_MOVABLE));
  22958. + void *p;
  22959. +
  22960. + p = kmem_cache_alloc(pool->handle_cachep,
  22961. + gfp & ~(__GFP_HIGHMEM|__GFP_MOVABLE));
  22962. +#ifdef CONFIG_PREEMPT_RT_FULL
  22963. + if (p) {
  22964. + struct zsmalloc_handle *zh = p;
  22965. +
  22966. + mutex_init(&zh->lock);
  22967. + }
  22968. +#endif
  22969. + return (unsigned long)p;
  22970. }
  22971. +#ifdef CONFIG_PREEMPT_RT_FULL
  22972. +static struct zsmalloc_handle *zs_get_pure_handle(unsigned long handle)
  22973. +{
  22974. + return (void *)(handle &~((1 << OBJ_TAG_BITS) - 1));
  22975. +}
  22976. +#endif
  22977. +
  22978. static void cache_free_handle(struct zs_pool *pool, unsigned long handle)
  22979. {
  22980. kmem_cache_free(pool->handle_cachep, (void *)handle);
  22981. @@ -373,12 +404,18 @@
  22982. static void record_obj(unsigned long handle, unsigned long obj)
  22983. {
  22984. +#ifdef CONFIG_PREEMPT_RT_FULL
  22985. + struct zsmalloc_handle *zh = zs_get_pure_handle(handle);
  22986. +
  22987. + WRITE_ONCE(zh->addr, obj);
  22988. +#else
  22989. /*
  22990. * lsb of @obj represents handle lock while other bits
  22991. * represent object value the handle is pointing so
  22992. * updating shouldn't do store tearing.
  22993. */
  22994. WRITE_ONCE(*(unsigned long *)handle, obj);
  22995. +#endif
  22996. }
  22997. /* zpool driver */
  22998. @@ -467,6 +504,7 @@
  22999. /* per-cpu VM mapping areas for zspage accesses that cross page boundaries */
  23000. static DEFINE_PER_CPU(struct mapping_area, zs_map_area);
  23001. +static DEFINE_LOCAL_IRQ_LOCK(zs_map_area_lock);
  23002. static bool is_zspage_isolated(struct zspage *zspage)
  23003. {
  23004. @@ -902,7 +940,13 @@
  23005. static unsigned long handle_to_obj(unsigned long handle)
  23006. {
  23007. +#ifdef CONFIG_PREEMPT_RT_FULL
  23008. + struct zsmalloc_handle *zh = zs_get_pure_handle(handle);
  23009. +
  23010. + return zh->addr;
  23011. +#else
  23012. return *(unsigned long *)handle;
  23013. +#endif
  23014. }
  23015. static unsigned long obj_to_head(struct page *page, void *obj)
  23016. @@ -916,22 +960,46 @@
  23017. static inline int testpin_tag(unsigned long handle)
  23018. {
  23019. +#ifdef CONFIG_PREEMPT_RT_FULL
  23020. + struct zsmalloc_handle *zh = zs_get_pure_handle(handle);
  23021. +
  23022. + return mutex_is_locked(&zh->lock);
  23023. +#else
  23024. return bit_spin_is_locked(HANDLE_PIN_BIT, (unsigned long *)handle);
  23025. +#endif
  23026. }
  23027. static inline int trypin_tag(unsigned long handle)
  23028. {
  23029. +#ifdef CONFIG_PREEMPT_RT_FULL
  23030. + struct zsmalloc_handle *zh = zs_get_pure_handle(handle);
  23031. +
  23032. + return mutex_trylock(&zh->lock);
  23033. +#else
  23034. return bit_spin_trylock(HANDLE_PIN_BIT, (unsigned long *)handle);
  23035. +#endif
  23036. }
  23037. static void pin_tag(unsigned long handle)
  23038. {
  23039. +#ifdef CONFIG_PREEMPT_RT_FULL
  23040. + struct zsmalloc_handle *zh = zs_get_pure_handle(handle);
  23041. +
  23042. + return mutex_lock(&zh->lock);
  23043. +#else
  23044. bit_spin_lock(HANDLE_PIN_BIT, (unsigned long *)handle);
  23045. +#endif
  23046. }
  23047. static void unpin_tag(unsigned long handle)
  23048. {
  23049. +#ifdef CONFIG_PREEMPT_RT_FULL
  23050. + struct zsmalloc_handle *zh = zs_get_pure_handle(handle);
  23051. +
  23052. + return mutex_unlock(&zh->lock);
  23053. +#else
  23054. bit_spin_unlock(HANDLE_PIN_BIT, (unsigned long *)handle);
  23055. +#endif
  23056. }
  23057. static void reset_page(struct page *page)
  23058. @@ -1423,7 +1491,7 @@
  23059. class = pool->size_class[class_idx];
  23060. off = (class->size * obj_idx) & ~PAGE_MASK;
  23061. - area = &get_cpu_var(zs_map_area);
  23062. + area = &get_locked_var(zs_map_area_lock, zs_map_area);
  23063. area->vm_mm = mm;
  23064. if (off + class->size <= PAGE_SIZE) {
  23065. /* this object is contained entirely within a page */
  23066. @@ -1477,7 +1545,7 @@
  23067. __zs_unmap_object(area, pages, off, class->size);
  23068. }
  23069. - put_cpu_var(zs_map_area);
  23070. + put_locked_var(zs_map_area_lock, zs_map_area);
  23071. migrate_read_unlock(zspage);
  23072. unpin_tag(handle);
  23073. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/core/dev.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/core/dev.c
  23074. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/core/dev.c 2017-04-16 10:38:31.000000000 +0200
  23075. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/core/dev.c 2017-04-18 17:54:26.000000000 +0200
  23076. @@ -190,6 +190,7 @@
  23077. static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
  23078. static seqcount_t devnet_rename_seq;
  23079. +static DEFINE_MUTEX(devnet_rename_mutex);
  23080. static inline void dev_base_seq_inc(struct net *net)
  23081. {
  23082. @@ -211,14 +212,14 @@
  23083. static inline void rps_lock(struct softnet_data *sd)
  23084. {
  23085. #ifdef CONFIG_RPS
  23086. - spin_lock(&sd->input_pkt_queue.lock);
  23087. + raw_spin_lock(&sd->input_pkt_queue.raw_lock);
  23088. #endif
  23089. }
  23090. static inline void rps_unlock(struct softnet_data *sd)
  23091. {
  23092. #ifdef CONFIG_RPS
  23093. - spin_unlock(&sd->input_pkt_queue.lock);
  23094. + raw_spin_unlock(&sd->input_pkt_queue.raw_lock);
  23095. #endif
  23096. }
  23097. @@ -888,7 +889,8 @@
  23098. strcpy(name, dev->name);
  23099. rcu_read_unlock();
  23100. if (read_seqcount_retry(&devnet_rename_seq, seq)) {
  23101. - cond_resched();
  23102. + mutex_lock(&devnet_rename_mutex);
  23103. + mutex_unlock(&devnet_rename_mutex);
  23104. goto retry;
  23105. }
  23106. @@ -1157,20 +1159,17 @@
  23107. if (dev->flags & IFF_UP)
  23108. return -EBUSY;
  23109. - write_seqcount_begin(&devnet_rename_seq);
  23110. + mutex_lock(&devnet_rename_mutex);
  23111. + __raw_write_seqcount_begin(&devnet_rename_seq);
  23112. - if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
  23113. - write_seqcount_end(&devnet_rename_seq);
  23114. - return 0;
  23115. - }
  23116. + if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
  23117. + goto outunlock;
  23118. memcpy(oldname, dev->name, IFNAMSIZ);
  23119. err = dev_get_valid_name(net, dev, newname);
  23120. - if (err < 0) {
  23121. - write_seqcount_end(&devnet_rename_seq);
  23122. - return err;
  23123. - }
  23124. + if (err < 0)
  23125. + goto outunlock;
  23126. if (oldname[0] && !strchr(oldname, '%'))
  23127. netdev_info(dev, "renamed from %s\n", oldname);
  23128. @@ -1183,11 +1182,12 @@
  23129. if (ret) {
  23130. memcpy(dev->name, oldname, IFNAMSIZ);
  23131. dev->name_assign_type = old_assign_type;
  23132. - write_seqcount_end(&devnet_rename_seq);
  23133. - return ret;
  23134. + err = ret;
  23135. + goto outunlock;
  23136. }
  23137. - write_seqcount_end(&devnet_rename_seq);
  23138. + __raw_write_seqcount_end(&devnet_rename_seq);
  23139. + mutex_unlock(&devnet_rename_mutex);
  23140. netdev_adjacent_rename_links(dev, oldname);
  23141. @@ -1208,7 +1208,8 @@
  23142. /* err >= 0 after dev_alloc_name() or stores the first errno */
  23143. if (err >= 0) {
  23144. err = ret;
  23145. - write_seqcount_begin(&devnet_rename_seq);
  23146. + mutex_lock(&devnet_rename_mutex);
  23147. + __raw_write_seqcount_begin(&devnet_rename_seq);
  23148. memcpy(dev->name, oldname, IFNAMSIZ);
  23149. memcpy(oldname, newname, IFNAMSIZ);
  23150. dev->name_assign_type = old_assign_type;
  23151. @@ -1221,6 +1222,11 @@
  23152. }
  23153. return err;
  23154. +
  23155. +outunlock:
  23156. + __raw_write_seqcount_end(&devnet_rename_seq);
  23157. + mutex_unlock(&devnet_rename_mutex);
  23158. + return err;
  23159. }
  23160. /**
  23161. @@ -2285,6 +2291,7 @@
  23162. sd->output_queue_tailp = &q->next_sched;
  23163. raise_softirq_irqoff(NET_TX_SOFTIRQ);
  23164. local_irq_restore(flags);
  23165. + preempt_check_resched_rt();
  23166. }
  23167. void __netif_schedule(struct Qdisc *q)
  23168. @@ -2366,6 +2373,7 @@
  23169. __this_cpu_write(softnet_data.completion_queue, skb);
  23170. raise_softirq_irqoff(NET_TX_SOFTIRQ);
  23171. local_irq_restore(flags);
  23172. + preempt_check_resched_rt();
  23173. }
  23174. EXPORT_SYMBOL(__dev_kfree_skb_irq);
  23175. @@ -3100,7 +3108,11 @@
  23176. * This permits qdisc->running owner to get the lock more
  23177. * often and dequeue packets faster.
  23178. */
  23179. +#ifdef CONFIG_PREEMPT_RT_FULL
  23180. + contended = true;
  23181. +#else
  23182. contended = qdisc_is_running(q);
  23183. +#endif
  23184. if (unlikely(contended))
  23185. spin_lock(&q->busylock);
  23186. @@ -3163,8 +3175,10 @@
  23187. #define skb_update_prio(skb)
  23188. #endif
  23189. +#ifndef CONFIG_PREEMPT_RT_FULL
  23190. DEFINE_PER_CPU(int, xmit_recursion);
  23191. EXPORT_SYMBOL(xmit_recursion);
  23192. +#endif
  23193. /**
  23194. * dev_loopback_xmit - loop back @skb
  23195. @@ -3398,8 +3412,7 @@
  23196. int cpu = smp_processor_id(); /* ok because BHs are off */
  23197. if (txq->xmit_lock_owner != cpu) {
  23198. - if (unlikely(__this_cpu_read(xmit_recursion) >
  23199. - XMIT_RECURSION_LIMIT))
  23200. + if (unlikely(xmit_rec_read() > XMIT_RECURSION_LIMIT))
  23201. goto recursion_alert;
  23202. skb = validate_xmit_skb(skb, dev);
  23203. @@ -3409,9 +3422,9 @@
  23204. HARD_TX_LOCK(dev, txq, cpu);
  23205. if (!netif_xmit_stopped(txq)) {
  23206. - __this_cpu_inc(xmit_recursion);
  23207. + xmit_rec_inc();
  23208. skb = dev_hard_start_xmit(skb, dev, txq, &rc);
  23209. - __this_cpu_dec(xmit_recursion);
  23210. + xmit_rec_dec();
  23211. if (dev_xmit_complete(rc)) {
  23212. HARD_TX_UNLOCK(dev, txq);
  23213. goto out;
  23214. @@ -3785,6 +3798,7 @@
  23215. rps_unlock(sd);
  23216. local_irq_restore(flags);
  23217. + preempt_check_resched_rt();
  23218. atomic_long_inc(&skb->dev->rx_dropped);
  23219. kfree_skb(skb);
  23220. @@ -3803,7 +3817,7 @@
  23221. struct rps_dev_flow voidflow, *rflow = &voidflow;
  23222. int cpu;
  23223. - preempt_disable();
  23224. + migrate_disable();
  23225. rcu_read_lock();
  23226. cpu = get_rps_cpu(skb->dev, skb, &rflow);
  23227. @@ -3813,13 +3827,13 @@
  23228. ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
  23229. rcu_read_unlock();
  23230. - preempt_enable();
  23231. + migrate_enable();
  23232. } else
  23233. #endif
  23234. {
  23235. unsigned int qtail;
  23236. - ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
  23237. - put_cpu();
  23238. + ret = enqueue_to_backlog(skb, get_cpu_light(), &qtail);
  23239. + put_cpu_light();
  23240. }
  23241. return ret;
  23242. }
  23243. @@ -3853,11 +3867,9 @@
  23244. trace_netif_rx_ni_entry(skb);
  23245. - preempt_disable();
  23246. + local_bh_disable();
  23247. err = netif_rx_internal(skb);
  23248. - if (local_softirq_pending())
  23249. - do_softirq();
  23250. - preempt_enable();
  23251. + local_bh_enable();
  23252. return err;
  23253. }
  23254. @@ -4336,7 +4348,7 @@
  23255. skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
  23256. if (skb->dev->reg_state == NETREG_UNREGISTERING) {
  23257. __skb_unlink(skb, &sd->input_pkt_queue);
  23258. - kfree_skb(skb);
  23259. + __skb_queue_tail(&sd->tofree_queue, skb);
  23260. input_queue_head_incr(sd);
  23261. }
  23262. }
  23263. @@ -4346,11 +4358,14 @@
  23264. skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
  23265. if (skb->dev->reg_state == NETREG_UNREGISTERING) {
  23266. __skb_unlink(skb, &sd->process_queue);
  23267. - kfree_skb(skb);
  23268. + __skb_queue_tail(&sd->tofree_queue, skb);
  23269. input_queue_head_incr(sd);
  23270. }
  23271. }
  23272. + if (!skb_queue_empty(&sd->tofree_queue))
  23273. + raise_softirq_irqoff(NET_RX_SOFTIRQ);
  23274. local_bh_enable();
  23275. +
  23276. }
  23277. static void flush_all_backlogs(void)
  23278. @@ -4831,6 +4846,7 @@
  23279. sd->rps_ipi_list = NULL;
  23280. local_irq_enable();
  23281. + preempt_check_resched_rt();
  23282. /* Send pending IPI's to kick RPS processing on remote cpus. */
  23283. while (remsd) {
  23284. @@ -4844,6 +4860,7 @@
  23285. } else
  23286. #endif
  23287. local_irq_enable();
  23288. + preempt_check_resched_rt();
  23289. }
  23290. static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
  23291. @@ -4873,7 +4890,9 @@
  23292. while (again) {
  23293. struct sk_buff *skb;
  23294. + local_irq_disable();
  23295. while ((skb = __skb_dequeue(&sd->process_queue))) {
  23296. + local_irq_enable();
  23297. rcu_read_lock();
  23298. __netif_receive_skb(skb);
  23299. rcu_read_unlock();
  23300. @@ -4881,9 +4900,9 @@
  23301. if (++work >= quota)
  23302. return work;
  23303. + local_irq_disable();
  23304. }
  23305. - local_irq_disable();
  23306. rps_lock(sd);
  23307. if (skb_queue_empty(&sd->input_pkt_queue)) {
  23308. /*
  23309. @@ -4921,9 +4940,11 @@
  23310. local_irq_save(flags);
  23311. ____napi_schedule(this_cpu_ptr(&softnet_data), n);
  23312. local_irq_restore(flags);
  23313. + preempt_check_resched_rt();
  23314. }
  23315. EXPORT_SYMBOL(__napi_schedule);
  23316. +#ifndef CONFIG_PREEMPT_RT_FULL
  23317. /**
  23318. * __napi_schedule_irqoff - schedule for receive
  23319. * @n: entry to schedule
  23320. @@ -4935,6 +4956,7 @@
  23321. ____napi_schedule(this_cpu_ptr(&softnet_data), n);
  23322. }
  23323. EXPORT_SYMBOL(__napi_schedule_irqoff);
  23324. +#endif
  23325. void __napi_complete(struct napi_struct *n)
  23326. {
  23327. @@ -5224,13 +5246,21 @@
  23328. struct softnet_data *sd = this_cpu_ptr(&softnet_data);
  23329. unsigned long time_limit = jiffies + 2;
  23330. int budget = netdev_budget;
  23331. + struct sk_buff_head tofree_q;
  23332. + struct sk_buff *skb;
  23333. LIST_HEAD(list);
  23334. LIST_HEAD(repoll);
  23335. + __skb_queue_head_init(&tofree_q);
  23336. +
  23337. local_irq_disable();
  23338. + skb_queue_splice_init(&sd->tofree_queue, &tofree_q);
  23339. list_splice_init(&sd->poll_list, &list);
  23340. local_irq_enable();
  23341. + while ((skb = __skb_dequeue(&tofree_q)))
  23342. + kfree_skb(skb);
  23343. +
  23344. for (;;) {
  23345. struct napi_struct *n;
  23346. @@ -5261,7 +5291,7 @@
  23347. list_splice_tail(&repoll, &list);
  23348. list_splice(&list, &sd->poll_list);
  23349. if (!list_empty(&sd->poll_list))
  23350. - __raise_softirq_irqoff(NET_RX_SOFTIRQ);
  23351. + __raise_softirq_irqoff_ksoft(NET_RX_SOFTIRQ);
  23352. net_rps_action_and_irq_enable(sd);
  23353. }
  23354. @@ -8022,16 +8052,20 @@
  23355. raise_softirq_irqoff(NET_TX_SOFTIRQ);
  23356. local_irq_enable();
  23357. + preempt_check_resched_rt();
  23358. /* Process offline CPU's input_pkt_queue */
  23359. while ((skb = __skb_dequeue(&oldsd->process_queue))) {
  23360. netif_rx_ni(skb);
  23361. input_queue_head_incr(oldsd);
  23362. }
  23363. - while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
  23364. + while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
  23365. netif_rx_ni(skb);
  23366. input_queue_head_incr(oldsd);
  23367. }
  23368. + while ((skb = __skb_dequeue(&oldsd->tofree_queue))) {
  23369. + kfree_skb(skb);
  23370. + }
  23371. return NOTIFY_OK;
  23372. }
  23373. @@ -8336,8 +8370,9 @@
  23374. INIT_WORK(flush, flush_backlog);
  23375. - skb_queue_head_init(&sd->input_pkt_queue);
  23376. - skb_queue_head_init(&sd->process_queue);
  23377. + skb_queue_head_init_raw(&sd->input_pkt_queue);
  23378. + skb_queue_head_init_raw(&sd->process_queue);
  23379. + skb_queue_head_init_raw(&sd->tofree_queue);
  23380. INIT_LIST_HEAD(&sd->poll_list);
  23381. sd->output_queue_tailp = &sd->output_queue;
  23382. #ifdef CONFIG_RPS
  23383. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/core/filter.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/core/filter.c
  23384. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/core/filter.c 2017-04-16 10:38:31.000000000 +0200
  23385. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/core/filter.c 2017-04-18 17:54:26.000000000 +0200
  23386. @@ -1645,7 +1645,7 @@
  23387. {
  23388. int ret;
  23389. - if (unlikely(__this_cpu_read(xmit_recursion) > XMIT_RECURSION_LIMIT)) {
  23390. + if (unlikely(xmit_rec_read() > XMIT_RECURSION_LIMIT)) {
  23391. net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
  23392. kfree_skb(skb);
  23393. return -ENETDOWN;
  23394. @@ -1653,9 +1653,9 @@
  23395. skb->dev = dev;
  23396. - __this_cpu_inc(xmit_recursion);
  23397. + xmit_rec_inc();
  23398. ret = dev_queue_xmit(skb);
  23399. - __this_cpu_dec(xmit_recursion);
  23400. + xmit_rec_dec();
  23401. return ret;
  23402. }
  23403. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/core/gen_estimator.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/core/gen_estimator.c
  23404. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/core/gen_estimator.c 2017-04-16 10:38:31.000000000 +0200
  23405. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/core/gen_estimator.c 2017-04-18 17:54:26.000000000 +0200
  23406. @@ -84,7 +84,7 @@
  23407. struct gnet_stats_basic_packed *bstats;
  23408. struct gnet_stats_rate_est64 *rate_est;
  23409. spinlock_t *stats_lock;
  23410. - seqcount_t *running;
  23411. + net_seqlock_t *running;
  23412. int ewma_log;
  23413. u32 last_packets;
  23414. unsigned long avpps;
  23415. @@ -213,7 +213,7 @@
  23416. struct gnet_stats_basic_cpu __percpu *cpu_bstats,
  23417. struct gnet_stats_rate_est64 *rate_est,
  23418. spinlock_t *stats_lock,
  23419. - seqcount_t *running,
  23420. + net_seqlock_t *running,
  23421. struct nlattr *opt)
  23422. {
  23423. struct gen_estimator *est;
  23424. @@ -309,7 +309,7 @@
  23425. struct gnet_stats_basic_cpu __percpu *cpu_bstats,
  23426. struct gnet_stats_rate_est64 *rate_est,
  23427. spinlock_t *stats_lock,
  23428. - seqcount_t *running, struct nlattr *opt)
  23429. + net_seqlock_t *running, struct nlattr *opt)
  23430. {
  23431. gen_kill_estimator(bstats, rate_est);
  23432. return gen_new_estimator(bstats, cpu_bstats, rate_est, stats_lock, running, opt);
  23433. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/core/gen_stats.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/core/gen_stats.c
  23434. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/core/gen_stats.c 2017-04-16 10:38:31.000000000 +0200
  23435. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/core/gen_stats.c 2017-04-18 17:54:26.000000000 +0200
  23436. @@ -130,7 +130,7 @@
  23437. }
  23438. void
  23439. -__gnet_stats_copy_basic(const seqcount_t *running,
  23440. +__gnet_stats_copy_basic(net_seqlock_t *running,
  23441. struct gnet_stats_basic_packed *bstats,
  23442. struct gnet_stats_basic_cpu __percpu *cpu,
  23443. struct gnet_stats_basic_packed *b)
  23444. @@ -143,10 +143,10 @@
  23445. }
  23446. do {
  23447. if (running)
  23448. - seq = read_seqcount_begin(running);
  23449. + seq = net_seq_begin(running);
  23450. bstats->bytes = b->bytes;
  23451. bstats->packets = b->packets;
  23452. - } while (running && read_seqcount_retry(running, seq));
  23453. + } while (running && net_seq_retry(running, seq));
  23454. }
  23455. EXPORT_SYMBOL(__gnet_stats_copy_basic);
  23456. @@ -164,7 +164,7 @@
  23457. * if the room in the socket buffer was not sufficient.
  23458. */
  23459. int
  23460. -gnet_stats_copy_basic(const seqcount_t *running,
  23461. +gnet_stats_copy_basic(net_seqlock_t *running,
  23462. struct gnet_dump *d,
  23463. struct gnet_stats_basic_cpu __percpu *cpu,
  23464. struct gnet_stats_basic_packed *b)
  23465. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/core/skbuff.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/core/skbuff.c
  23466. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/core/skbuff.c 2017-04-16 10:38:31.000000000 +0200
  23467. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/core/skbuff.c 2017-04-18 17:54:26.000000000 +0200
  23468. @@ -64,6 +64,7 @@
  23469. #include <linux/errqueue.h>
  23470. #include <linux/prefetch.h>
  23471. #include <linux/if_vlan.h>
  23472. +#include <linux/locallock.h>
  23473. #include <net/protocol.h>
  23474. #include <net/dst.h>
  23475. @@ -360,6 +361,8 @@
  23476. static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
  23477. static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
  23478. +static DEFINE_LOCAL_IRQ_LOCK(netdev_alloc_lock);
  23479. +static DEFINE_LOCAL_IRQ_LOCK(napi_alloc_cache_lock);
  23480. static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
  23481. {
  23482. @@ -367,10 +370,10 @@
  23483. unsigned long flags;
  23484. void *data;
  23485. - local_irq_save(flags);
  23486. + local_lock_irqsave(netdev_alloc_lock, flags);
  23487. nc = this_cpu_ptr(&netdev_alloc_cache);
  23488. data = __alloc_page_frag(nc, fragsz, gfp_mask);
  23489. - local_irq_restore(flags);
  23490. + local_unlock_irqrestore(netdev_alloc_lock, flags);
  23491. return data;
  23492. }
  23493. @@ -389,9 +392,13 @@
  23494. static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
  23495. {
  23496. - struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
  23497. + struct napi_alloc_cache *nc;
  23498. + void *data;
  23499. - return __alloc_page_frag(&nc->page, fragsz, gfp_mask);
  23500. + nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  23501. + data = __alloc_page_frag(&nc->page, fragsz, gfp_mask);
  23502. + put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  23503. + return data;
  23504. }
  23505. void *napi_alloc_frag(unsigned int fragsz)
  23506. @@ -438,13 +445,13 @@
  23507. if (sk_memalloc_socks())
  23508. gfp_mask |= __GFP_MEMALLOC;
  23509. - local_irq_save(flags);
  23510. + local_lock_irqsave(netdev_alloc_lock, flags);
  23511. nc = this_cpu_ptr(&netdev_alloc_cache);
  23512. data = __alloc_page_frag(nc, len, gfp_mask);
  23513. pfmemalloc = nc->pfmemalloc;
  23514. - local_irq_restore(flags);
  23515. + local_unlock_irqrestore(netdev_alloc_lock, flags);
  23516. if (unlikely(!data))
  23517. return NULL;
  23518. @@ -485,9 +492,10 @@
  23519. struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
  23520. gfp_t gfp_mask)
  23521. {
  23522. - struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
  23523. + struct napi_alloc_cache *nc;
  23524. struct sk_buff *skb;
  23525. void *data;
  23526. + bool pfmemalloc;
  23527. len += NET_SKB_PAD + NET_IP_ALIGN;
  23528. @@ -505,7 +513,10 @@
  23529. if (sk_memalloc_socks())
  23530. gfp_mask |= __GFP_MEMALLOC;
  23531. + nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  23532. data = __alloc_page_frag(&nc->page, len, gfp_mask);
  23533. + pfmemalloc = nc->page.pfmemalloc;
  23534. + put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  23535. if (unlikely(!data))
  23536. return NULL;
  23537. @@ -516,7 +527,7 @@
  23538. }
  23539. /* use OR instead of assignment to avoid clearing of bits in mask */
  23540. - if (nc->page.pfmemalloc)
  23541. + if (pfmemalloc)
  23542. skb->pfmemalloc = 1;
  23543. skb->head_frag = 1;
  23544. @@ -760,23 +771,26 @@
  23545. void __kfree_skb_flush(void)
  23546. {
  23547. - struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
  23548. + struct napi_alloc_cache *nc;
  23549. + nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  23550. /* flush skb_cache if containing objects */
  23551. if (nc->skb_count) {
  23552. kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count,
  23553. nc->skb_cache);
  23554. nc->skb_count = 0;
  23555. }
  23556. + put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  23557. }
  23558. static inline void _kfree_skb_defer(struct sk_buff *skb)
  23559. {
  23560. - struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
  23561. + struct napi_alloc_cache *nc;
  23562. /* drop skb->head and call any destructors for packet */
  23563. skb_release_all(skb);
  23564. + nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  23565. /* record skb to CPU local list */
  23566. nc->skb_cache[nc->skb_count++] = skb;
  23567. @@ -791,6 +805,7 @@
  23568. nc->skb_cache);
  23569. nc->skb_count = 0;
  23570. }
  23571. + put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
  23572. }
  23573. void __kfree_skb_defer(struct sk_buff *skb)
  23574. {
  23575. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/core/sock.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/core/sock.c
  23576. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/core/sock.c 2017-04-16 10:38:31.000000000 +0200
  23577. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/core/sock.c 2017-04-18 17:54:26.000000000 +0200
  23578. @@ -2499,12 +2499,11 @@
  23579. if (sk->sk_lock.owned)
  23580. __lock_sock(sk);
  23581. sk->sk_lock.owned = 1;
  23582. - spin_unlock(&sk->sk_lock.slock);
  23583. + spin_unlock_bh(&sk->sk_lock.slock);
  23584. /*
  23585. * The sk_lock has mutex_lock() semantics here:
  23586. */
  23587. mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
  23588. - local_bh_enable();
  23589. }
  23590. EXPORT_SYMBOL(lock_sock_nested);
  23591. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/ipv4/icmp.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/ipv4/icmp.c
  23592. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/ipv4/icmp.c 2017-04-16 10:38:31.000000000 +0200
  23593. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/ipv4/icmp.c 2017-04-18 17:54:26.000000000 +0200
  23594. @@ -69,6 +69,7 @@
  23595. #include <linux/jiffies.h>
  23596. #include <linux/kernel.h>
  23597. #include <linux/fcntl.h>
  23598. +#include <linux/sysrq.h>
  23599. #include <linux/socket.h>
  23600. #include <linux/in.h>
  23601. #include <linux/inet.h>
  23602. @@ -77,6 +78,7 @@
  23603. #include <linux/string.h>
  23604. #include <linux/netfilter_ipv4.h>
  23605. #include <linux/slab.h>
  23606. +#include <linux/locallock.h>
  23607. #include <net/snmp.h>
  23608. #include <net/ip.h>
  23609. #include <net/route.h>
  23610. @@ -204,6 +206,8 @@
  23611. *
  23612. * On SMP we have one ICMP socket per-cpu.
  23613. */
  23614. +static DEFINE_LOCAL_IRQ_LOCK(icmp_sk_lock);
  23615. +
  23616. static struct sock *icmp_sk(struct net *net)
  23617. {
  23618. return *this_cpu_ptr(net->ipv4.icmp_sk);
  23619. @@ -215,12 +219,14 @@
  23620. local_bh_disable();
  23621. + local_lock(icmp_sk_lock);
  23622. sk = icmp_sk(net);
  23623. if (unlikely(!spin_trylock(&sk->sk_lock.slock))) {
  23624. /* This can happen if the output path signals a
  23625. * dst_link_failure() for an outgoing ICMP packet.
  23626. */
  23627. + local_unlock(icmp_sk_lock);
  23628. local_bh_enable();
  23629. return NULL;
  23630. }
  23631. @@ -230,6 +236,7 @@
  23632. static inline void icmp_xmit_unlock(struct sock *sk)
  23633. {
  23634. spin_unlock_bh(&sk->sk_lock.slock);
  23635. + local_unlock(icmp_sk_lock);
  23636. }
  23637. int sysctl_icmp_msgs_per_sec __read_mostly = 1000;
  23638. @@ -358,6 +365,7 @@
  23639. struct sock *sk;
  23640. struct sk_buff *skb;
  23641. + local_lock(icmp_sk_lock);
  23642. sk = icmp_sk(dev_net((*rt)->dst.dev));
  23643. if (ip_append_data(sk, fl4, icmp_glue_bits, icmp_param,
  23644. icmp_param->data_len+icmp_param->head_len,
  23645. @@ -380,6 +388,7 @@
  23646. skb->ip_summed = CHECKSUM_NONE;
  23647. ip_push_pending_frames(sk, fl4);
  23648. }
  23649. + local_unlock(icmp_sk_lock);
  23650. }
  23651. /*
  23652. @@ -891,6 +900,30 @@
  23653. }
  23654. /*
  23655. + * 32bit and 64bit have different timestamp length, so we check for
  23656. + * the cookie at offset 20 and verify it is repeated at offset 50
  23657. + */
  23658. +#define CO_POS0 20
  23659. +#define CO_POS1 50
  23660. +#define CO_SIZE sizeof(int)
  23661. +#define ICMP_SYSRQ_SIZE 57
  23662. +
  23663. +/*
  23664. + * We got a ICMP_SYSRQ_SIZE sized ping request. Check for the cookie
  23665. + * pattern and if it matches send the next byte as a trigger to sysrq.
  23666. + */
  23667. +static void icmp_check_sysrq(struct net *net, struct sk_buff *skb)
  23668. +{
  23669. + int cookie = htonl(net->ipv4.sysctl_icmp_echo_sysrq);
  23670. + char *p = skb->data;
  23671. +
  23672. + if (!memcmp(&cookie, p + CO_POS0, CO_SIZE) &&
  23673. + !memcmp(&cookie, p + CO_POS1, CO_SIZE) &&
  23674. + p[CO_POS0 + CO_SIZE] == p[CO_POS1 + CO_SIZE])
  23675. + handle_sysrq(p[CO_POS0 + CO_SIZE]);
  23676. +}
  23677. +
  23678. +/*
  23679. * Handle ICMP_ECHO ("ping") requests.
  23680. *
  23681. * RFC 1122: 3.2.2.6 MUST have an echo server that answers ICMP echo
  23682. @@ -917,6 +950,11 @@
  23683. icmp_param.data_len = skb->len;
  23684. icmp_param.head_len = sizeof(struct icmphdr);
  23685. icmp_reply(&icmp_param, skb);
  23686. +
  23687. + if (skb->len == ICMP_SYSRQ_SIZE &&
  23688. + net->ipv4.sysctl_icmp_echo_sysrq) {
  23689. + icmp_check_sysrq(net, skb);
  23690. + }
  23691. }
  23692. /* should there be an ICMP stat for ignored echos? */
  23693. return true;
  23694. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/ipv4/sysctl_net_ipv4.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/ipv4/sysctl_net_ipv4.c
  23695. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/ipv4/sysctl_net_ipv4.c 2017-04-16 10:38:31.000000000 +0200
  23696. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/ipv4/sysctl_net_ipv4.c 2017-04-18 17:54:27.000000000 +0200
  23697. @@ -681,6 +681,13 @@
  23698. .proc_handler = proc_dointvec
  23699. },
  23700. {
  23701. + .procname = "icmp_echo_sysrq",
  23702. + .data = &init_net.ipv4.sysctl_icmp_echo_sysrq,
  23703. + .maxlen = sizeof(int),
  23704. + .mode = 0644,
  23705. + .proc_handler = proc_dointvec
  23706. + },
  23707. + {
  23708. .procname = "icmp_ignore_bogus_error_responses",
  23709. .data = &init_net.ipv4.sysctl_icmp_ignore_bogus_error_responses,
  23710. .maxlen = sizeof(int),
  23711. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/ipv4/tcp_ipv4.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/ipv4/tcp_ipv4.c
  23712. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/ipv4/tcp_ipv4.c 2017-04-16 10:38:31.000000000 +0200
  23713. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/ipv4/tcp_ipv4.c 2017-04-18 17:54:27.000000000 +0200
  23714. @@ -62,6 +62,7 @@
  23715. #include <linux/init.h>
  23716. #include <linux/times.h>
  23717. #include <linux/slab.h>
  23718. +#include <linux/locallock.h>
  23719. #include <net/net_namespace.h>
  23720. #include <net/icmp.h>
  23721. @@ -568,6 +569,7 @@
  23722. }
  23723. EXPORT_SYMBOL(tcp_v4_send_check);
  23724. +static DEFINE_LOCAL_IRQ_LOCK(tcp_sk_lock);
  23725. /*
  23726. * This routine will send an RST to the other tcp.
  23727. *
  23728. @@ -695,6 +697,8 @@
  23729. offsetof(struct inet_timewait_sock, tw_bound_dev_if));
  23730. arg.tos = ip_hdr(skb)->tos;
  23731. +
  23732. + local_lock(tcp_sk_lock);
  23733. local_bh_disable();
  23734. ip_send_unicast_reply(*this_cpu_ptr(net->ipv4.tcp_sk),
  23735. skb, &TCP_SKB_CB(skb)->header.h4.opt,
  23736. @@ -704,6 +708,7 @@
  23737. __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
  23738. __TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
  23739. local_bh_enable();
  23740. + local_unlock(tcp_sk_lock);
  23741. #ifdef CONFIG_TCP_MD5SIG
  23742. out:
  23743. @@ -779,6 +784,7 @@
  23744. if (oif)
  23745. arg.bound_dev_if = oif;
  23746. arg.tos = tos;
  23747. + local_lock(tcp_sk_lock);
  23748. local_bh_disable();
  23749. ip_send_unicast_reply(*this_cpu_ptr(net->ipv4.tcp_sk),
  23750. skb, &TCP_SKB_CB(skb)->header.h4.opt,
  23751. @@ -787,6 +793,7 @@
  23752. __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
  23753. local_bh_enable();
  23754. + local_unlock(tcp_sk_lock);
  23755. }
  23756. static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb)
  23757. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/mac80211/rx.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/mac80211/rx.c
  23758. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/mac80211/rx.c 2017-04-16 10:38:32.000000000 +0200
  23759. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/mac80211/rx.c 2017-04-18 17:54:27.000000000 +0200
  23760. @@ -4180,7 +4180,7 @@
  23761. struct ieee80211_supported_band *sband;
  23762. struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
  23763. - WARN_ON_ONCE(softirq_count() == 0);
  23764. + WARN_ON_ONCE_NONRT(softirq_count() == 0);
  23765. if (WARN_ON(status->band >= NUM_NL80211_BANDS))
  23766. goto drop;
  23767. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/netfilter/core.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/netfilter/core.c
  23768. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/netfilter/core.c 2017-04-16 10:38:32.000000000 +0200
  23769. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/netfilter/core.c 2017-04-18 17:54:27.000000000 +0200
  23770. @@ -22,12 +22,18 @@
  23771. #include <linux/proc_fs.h>
  23772. #include <linux/mutex.h>
  23773. #include <linux/slab.h>
  23774. +#include <linux/locallock.h>
  23775. #include <linux/rcupdate.h>
  23776. #include <net/net_namespace.h>
  23777. #include <net/sock.h>
  23778. #include "nf_internals.h"
  23779. +#ifdef CONFIG_PREEMPT_RT_BASE
  23780. +DEFINE_LOCAL_IRQ_LOCK(xt_write_lock);
  23781. +EXPORT_PER_CPU_SYMBOL(xt_write_lock);
  23782. +#endif
  23783. +
  23784. static DEFINE_MUTEX(afinfo_mutex);
  23785. const struct nf_afinfo __rcu *nf_afinfo[NFPROTO_NUMPROTO] __read_mostly;
  23786. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/packet/af_packet.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/packet/af_packet.c
  23787. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/packet/af_packet.c 2017-04-16 10:38:33.000000000 +0200
  23788. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/packet/af_packet.c 2017-04-18 17:54:27.000000000 +0200
  23789. @@ -63,6 +63,7 @@
  23790. #include <linux/if_packet.h>
  23791. #include <linux/wireless.h>
  23792. #include <linux/kernel.h>
  23793. +#include <linux/delay.h>
  23794. #include <linux/kmod.h>
  23795. #include <linux/slab.h>
  23796. #include <linux/vmalloc.h>
  23797. @@ -694,7 +695,7 @@
  23798. if (BLOCK_NUM_PKTS(pbd)) {
  23799. while (atomic_read(&pkc->blk_fill_in_prog)) {
  23800. /* Waiting for skb_copy_bits to finish... */
  23801. - cpu_relax();
  23802. + cpu_chill();
  23803. }
  23804. }
  23805. @@ -956,7 +957,7 @@
  23806. if (!(status & TP_STATUS_BLK_TMO)) {
  23807. while (atomic_read(&pkc->blk_fill_in_prog)) {
  23808. /* Waiting for skb_copy_bits to finish... */
  23809. - cpu_relax();
  23810. + cpu_chill();
  23811. }
  23812. }
  23813. prb_close_block(pkc, pbd, po, status);
  23814. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/rds/ib_rdma.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/rds/ib_rdma.c
  23815. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/rds/ib_rdma.c 2017-04-16 10:38:33.000000000 +0200
  23816. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/rds/ib_rdma.c 2017-04-18 17:54:27.000000000 +0200
  23817. @@ -34,6 +34,7 @@
  23818. #include <linux/slab.h>
  23819. #include <linux/rculist.h>
  23820. #include <linux/llist.h>
  23821. +#include <linux/delay.h>
  23822. #include "rds_single_path.h"
  23823. #include "ib_mr.h"
  23824. @@ -210,7 +211,7 @@
  23825. for_each_online_cpu(cpu) {
  23826. flag = &per_cpu(clean_list_grace, cpu);
  23827. while (test_bit(CLEAN_LIST_BUSY_BIT, flag))
  23828. - cpu_relax();
  23829. + cpu_chill();
  23830. }
  23831. }
  23832. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/rxrpc/security.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/rxrpc/security.c
  23833. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/rxrpc/security.c 2017-04-16 10:38:33.000000000 +0200
  23834. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/rxrpc/security.c 2017-04-18 17:54:27.000000000 +0200
  23835. @@ -19,9 +19,6 @@
  23836. #include <keys/rxrpc-type.h>
  23837. #include "ar-internal.h"
  23838. -static LIST_HEAD(rxrpc_security_methods);
  23839. -static DECLARE_RWSEM(rxrpc_security_sem);
  23840. -
  23841. static const struct rxrpc_security *rxrpc_security_types[] = {
  23842. [RXRPC_SECURITY_NONE] = &rxrpc_no_security,
  23843. #ifdef CONFIG_RXKAD
  23844. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/sched/sch_api.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/sched/sch_api.c
  23845. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/sched/sch_api.c 2017-04-16 10:38:33.000000000 +0200
  23846. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/sched/sch_api.c 2017-04-18 17:54:27.000000000 +0200
  23847. @@ -981,7 +981,7 @@
  23848. rcu_assign_pointer(sch->stab, stab);
  23849. }
  23850. if (tca[TCA_RATE]) {
  23851. - seqcount_t *running;
  23852. + net_seqlock_t *running;
  23853. err = -EOPNOTSUPP;
  23854. if (sch->flags & TCQ_F_MQROOT)
  23855. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/sched/sch_generic.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/sched/sch_generic.c
  23856. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/sched/sch_generic.c 2017-04-16 10:38:33.000000000 +0200
  23857. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/sched/sch_generic.c 2017-04-18 17:54:27.000000000 +0200
  23858. @@ -425,7 +425,11 @@
  23859. .ops = &noop_qdisc_ops,
  23860. .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
  23861. .dev_queue = &noop_netdev_queue,
  23862. +#ifdef CONFIG_PREEMPT_RT_BASE
  23863. + .running = __SEQLOCK_UNLOCKED(noop_qdisc.running),
  23864. +#else
  23865. .running = SEQCNT_ZERO(noop_qdisc.running),
  23866. +#endif
  23867. .busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
  23868. };
  23869. EXPORT_SYMBOL(noop_qdisc);
  23870. @@ -624,9 +628,17 @@
  23871. lockdep_set_class(&sch->busylock,
  23872. dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
  23873. +#ifdef CONFIG_PREEMPT_RT_BASE
  23874. + seqlock_init(&sch->running);
  23875. + lockdep_set_class(&sch->running.seqcount,
  23876. + dev->qdisc_running_key ?: &qdisc_running_key);
  23877. + lockdep_set_class(&sch->running.lock,
  23878. + dev->qdisc_running_key ?: &qdisc_running_key);
  23879. +#else
  23880. seqcount_init(&sch->running);
  23881. lockdep_set_class(&sch->running,
  23882. dev->qdisc_running_key ?: &qdisc_running_key);
  23883. +#endif
  23884. sch->ops = ops;
  23885. sch->enqueue = ops->enqueue;
  23886. @@ -925,7 +937,7 @@
  23887. /* Wait for outstanding qdisc_run calls. */
  23888. list_for_each_entry(dev, head, close_list)
  23889. while (some_qdisc_is_busy(dev))
  23890. - yield();
  23891. + msleep(1);
  23892. }
  23893. void dev_deactivate(struct net_device *dev)
  23894. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/sunrpc/svc_xprt.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/sunrpc/svc_xprt.c
  23895. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/net/sunrpc/svc_xprt.c 2017-04-16 10:38:34.000000000 +0200
  23896. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/net/sunrpc/svc_xprt.c 2017-04-18 17:54:27.000000000 +0200
  23897. @@ -396,7 +396,7 @@
  23898. goto out;
  23899. }
  23900. - cpu = get_cpu();
  23901. + cpu = get_cpu_light();
  23902. pool = svc_pool_for_cpu(xprt->xpt_server, cpu);
  23903. atomic_long_inc(&pool->sp_stats.packets);
  23904. @@ -432,7 +432,7 @@
  23905. atomic_long_inc(&pool->sp_stats.threads_woken);
  23906. wake_up_process(rqstp->rq_task);
  23907. - put_cpu();
  23908. + put_cpu_light();
  23909. goto out;
  23910. }
  23911. rcu_read_unlock();
  23912. @@ -453,7 +453,7 @@
  23913. goto redo_search;
  23914. }
  23915. rqstp = NULL;
  23916. - put_cpu();
  23917. + put_cpu_light();
  23918. out:
  23919. trace_svc_xprt_do_enqueue(xprt, rqstp);
  23920. }
  23921. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/scripts/mkcompile_h linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/scripts/mkcompile_h
  23922. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/scripts/mkcompile_h 2017-04-16 10:38:34.000000000 +0200
  23923. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/scripts/mkcompile_h 2017-04-18 17:54:27.000000000 +0200
  23924. @@ -4,7 +4,8 @@
  23925. ARCH=$2
  23926. SMP=$3
  23927. PREEMPT=$4
  23928. -CC=$5
  23929. +RT=$5
  23930. +CC=$6
  23931. vecho() { [ "${quiet}" = "silent_" ] || echo "$@" ; }
  23932. @@ -57,6 +58,7 @@
  23933. CONFIG_FLAGS=""
  23934. if [ -n "$SMP" ] ; then CONFIG_FLAGS="SMP"; fi
  23935. if [ -n "$PREEMPT" ] ; then CONFIG_FLAGS="$CONFIG_FLAGS PREEMPT"; fi
  23936. +if [ -n "$RT" ] ; then CONFIG_FLAGS="$CONFIG_FLAGS RT"; fi
  23937. UTS_VERSION="$UTS_VERSION $CONFIG_FLAGS $TIMESTAMP"
  23938. # Truncate to maximum length
  23939. diff -Nur linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/sound/core/pcm_native.c linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/sound/core/pcm_native.c
  23940. --- linux-72134397d72079a533c8fc742701fdc7f5ae7c5b.orig/sound/core/pcm_native.c 2017-04-16 10:38:35.000000000 +0200
  23941. +++ linux-72134397d72079a533c8fc742701fdc7f5ae7c5b/sound/core/pcm_native.c 2017-04-18 17:54:27.000000000 +0200
  23942. @@ -135,7 +135,7 @@
  23943. void snd_pcm_stream_lock_irq(struct snd_pcm_substream *substream)
  23944. {
  23945. if (!substream->pcm->nonatomic)
  23946. - local_irq_disable();
  23947. + local_irq_disable_nort();
  23948. snd_pcm_stream_lock(substream);
  23949. }
  23950. EXPORT_SYMBOL_GPL(snd_pcm_stream_lock_irq);
  23951. @@ -150,7 +150,7 @@
  23952. {
  23953. snd_pcm_stream_unlock(substream);
  23954. if (!substream->pcm->nonatomic)
  23955. - local_irq_enable();
  23956. + local_irq_enable_nort();
  23957. }
  23958. EXPORT_SYMBOL_GPL(snd_pcm_stream_unlock_irq);
  23959. @@ -158,7 +158,7 @@
  23960. {
  23961. unsigned long flags = 0;
  23962. if (!substream->pcm->nonatomic)
  23963. - local_irq_save(flags);
  23964. + local_irq_save_nort(flags);
  23965. snd_pcm_stream_lock(substream);
  23966. return flags;
  23967. }
  23968. @@ -176,7 +176,7 @@
  23969. {
  23970. snd_pcm_stream_unlock(substream);
  23971. if (!substream->pcm->nonatomic)
  23972. - local_irq_restore(flags);
  23973. + local_irq_restore_nort(flags);
  23974. }
  23975. EXPORT_SYMBOL_GPL(snd_pcm_stream_unlock_irqrestore);